使用 Garage 构建轻量级自托管 S3 对象存储

下载

  1. 下载Garage
1
curl -LO https://garagehq.deuxfleurs.fr/_releases/v2.5.0/x86_64-unknown-linux-musl/garage
  1. 安装Garage
1
2
3
chmod +x ./garage

sudo mv ./garage /usr/local/bin

配置

  1. 创建存储目录
1
2
sudo mkdir -p /var/lib/garage/meta /var/lib/garage/data
sudo chown -R $USER:$USER /var/lib/garage
  1. 创建配置文件

配置文件路径默认是/etc/garage.toml,可以改成其他路径,设置好环境变量GARAGE_CONFIG_FILE,或者每次执行命令添加-c /path/to/garage.toml

1
export GARAGE_CONFIG_FILE=/etc/garage.toml

配置内容如下

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
sudo tee /etc/garage.toml >/dev/null <<EOF
metadata_dir = "/var/lib/garage/meta"
data_dir = "/var/lib/garage/data"

# 单机测试/起步可用 sqlite;多节点且 replication_factor >= 2 推荐 lmdb
db_engine = "sqlite"
# 副本数不能超过节点数;单机/测试用 1,生产多节点常用 3(需对应节点与 zone 规划)
replication_factor = 1

# 若改用 lmdb,开启自动元数据快照。注意:lmdb 不可跨 CPU 架构迁移
# db_engine = "lmdb"
# metadata_auto_snapshot_interval = "6h"
# compression_level = 2

# 节点间 RPC 通信监听地址(默认 3901),仅用于集群内部节点通信与数据同步,勿直接暴露公网
rpc_bind_addr = "[::]:3901"
# 单机可填 127.0.0.1:3901;多节点集群必须填其他节点能连上的本机 IP/域名:3901
rpc_public_addr = "127.0.0.1:3901"
# 集群内所有节点必须使用相同的 rpc_secret
rpc_secret = "$(openssl rand -hex 32)"

# S3 标准协议端点(供后端 SDK、CLI 工具、预签名上传及对象管理使用)
[s3_api]
s3_region = "garage"
api_bind_addr = "[::]:3900"
root_domain = ".s3.mydomain.com"
# 本地测试可用 http://127.0.0.1:3900,nginx用 https://s3.mydomain.com
public_endpoint = "http://127.0.0.1:3900"

# S3 静态网站/公开 Bucket 直访端点(无需 S3 签名,浏览器可直接访问)
[s3_web]
bind_addr = "[::]:3902"
root_domain = ".cdn.mydomain.com"
index = "index.html"

# 运维管理与监控接口,用于集群状态查询、健康检查与指标采集,切勿直接暴露公网
# 各节点均可开启(保持一致的 token,便于本地/内网管理与监控;切勿直接暴露公网)
[admin]
api_bind_addr = "[::]:3903"
admin_token = "$(openssl rand -base64 32)"
metrics_token = "$(openssl rand -base64 32)"
EOF

启动服务器

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
# export GARAGE_CONFIG_FILE=/etc/garage.toml
garage server

# 2026-11-20T03:16:02.232638Z INFO garage::server: Loading configuration from /etc/garage.toml...
# 2026-11-20T03:16:02.235128Z INFO garage::server: Initializing Garage main data store...
# 2026-11-20T03:16:02.243008Z INFO garage_model::garage: Opening database...
# 2026-11-20T03:16:02.248581Z INFO garage_db::sqlite_adapter: Opening Sqlite database at: /var/lib/garage/meta/db.sqlite
# 2026-11-20T03:16:02.261430Z INFO garage_model::garage: Initializing RPC...
# 2026-11-20T03:16:02.261555Z INFO garage_model::garage: Initialize background variable system...
# 2026-11-20T03:16:02.261571Z INFO garage_model::garage: Initialize membership management system...
# 2026-11-20T03:16:02.261632Z INFO garage_rpc::system: Generating new node key pair.
# 2026-11-20T03:16:02.263862Z INFO garage_rpc::system: Node ID of this node: 5c81343aa6d99fde
# 2026-11-20T03:16:02.264452Z INFO garage_rpc::layout::manager: No valid previous cluster layout stored (IO error: No such file or directory (os error 2)), starting fresh.
# 2026-11-20T03:16:02.264859Z INFO garage_rpc::layout::helper: ack_until updated to 0
# 2026-11-20T03:16:02.266518Z INFO garage_model::garage: Initialize block manager...
# 2026-11-20T03:16:02.272119Z INFO garage_model::garage: Initialize admin_token_table...
# 2026-11-20T03:16:02.273241Z INFO garage_model::garage: Initialize bucket_table...
# 2026-11-20T03:16:02.274183Z INFO garage_model::garage: Initialize bucket_alias_table...
# 2026-11-20T03:16:02.275431Z INFO garage_model::garage: Initialize key_table_table...
# 2026-11-20T03:16:02.276746Z INFO garage_model::garage: Initialize block_ref_table...
# 2026-11-20T03:16:02.279925Z INFO garage_model::garage: Initialize version_table...
# 2026-11-20T03:16:02.282167Z INFO garage_model::garage: Initialize multipart upload counter table...
# 2026-11-20T03:16:02.285473Z INFO garage_model::garage: Initialize multipart upload table...
# 2026-11-20T03:16:02.287074Z INFO garage_model::garage: Initialize object counter table...
# 2026-11-20T03:16:02.288194Z INFO garage_model::garage: Initialize object_table...
# 2026-11-20T03:16:02.289329Z INFO garage_model::garage: Load lifecycle worker state...
# 2026-11-20T03:16:02.289438Z INFO garage_model::garage: Initialize K2V counter table...
# 2026-11-20T03:16:02.290522Z INFO garage_model::garage: Initialize K2V subscription manager...
# 2026-11-20T03:16:02.290554Z INFO garage_model::garage: Initialize K2V item table...
# 2026-11-20T03:16:02.291615Z INFO garage_model::garage: Initialize K2V RPC handler...
# 2026-11-20T03:16:02.291907Z INFO garage::server: Initializing background runner...
# 2026-11-20T03:16:02.291987Z INFO garage::server: Spawning Garage workers...
# 2026-11-20T03:16:02.292175Z INFO garage_model::s3::lifecycle_worker: Starting lifecycle worker for 2026-11-20
# 2026-11-20T03:16:02.292187Z INFO garage::server: Initialize Admin API server and metrics collector...
# 2026-11-20T03:16:02.294265Z INFO garage_model::s3::lifecycle_worker: Lifecycle worker finished for 2026-11-20, objects expired: 0, mpu aborted: 0
# 2026-11-20T03:16:02.409184Z INFO garage::server: Launching internal Garage cluster communications...
# 2026-11-20T03:16:02.409250Z INFO garage::server: Initializing S3 API server...
# 2026-11-20T03:16:02.409259Z INFO garage::server: Launching Admin API server...
# 2026-11-20T03:16:02.409352Z INFO garage_api_common::generic_server: Admin API server listening on http://[::]:3903
# 2026-11-20T03:16:02.412065Z INFO garage_api_common::generic_server: S3 API server listening on http://[::]:3900
# 2026-11-20T03:16:02.425717Z INFO garage_net::netapp: Listening on [::]:3901

核对状态

1
2
3
4
5
6
# 查看节点ID(记下前几位或完整ID)
garage status

# ==== HEALTHY NODES ====
# ID Hostname Address Tags Zone Capacity DataAvail ResyncQueue Version
# 5c81343aa6d99fde DESKTOP-97FP0NF 127.0.0.1:3901 NO ROLE ASSIGNED 0 v2.5.0

分配layout

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
# 1. 暂存角色分配
garage layout assign 5c81343aa6d99fde -z dc1 -c 10G

# Role changes are staged but not yet committed.
# Use `garage layout show` to view staged role changes,
# and `garage layout apply` to enact staged changes.

# 2. 检查暂存变更是否正确
garage layout show

# ==== CURRENT CLUSTER LAYOUT ====
# No nodes currently have a role in the cluster.
# See `garage status` to view available nodes.

# Current cluster layout version: 0

# ==== STAGED ROLE CHANGES ====
# ID Tags Zone Capacity
# 5c81343aa6d99fde [] dc1 9.3 GiB


# ==== NEW CLUSTER LAYOUT AFTER APPLYING CHANGES ====
# ID Tags Zone Capacity Usable capacity
# 5c81343aa6d99fde [] dc1 9.3 GiB 9.3 GiB (100.0%)

# Zone redundancy: maximum

# ==== COMPUTATION OF A NEW PARTITION ASSIGNATION ====

# Partitions are replicated 1 times on at least 1 distinct zones.

# Optimal partition size: 37.3 MiB
# Usable capacity / total cluster capacity: 9.3 GiB / 9.3 GiB (100.0 %)
# Effective capacity (replication factor 1): 9.3 GiB

# dc1 Tags Partitions Capacity Usable capacity
# 5c81343aa6d99fde [] 256 (256 new) 9.3 GiB 9.3 GiB (100.0%)
# TOTAL 256 (256 unique) 9.3 GiB 9.3 GiB (100.0%)


# To enact the staged role changes, type:

# garage layout apply --version 1

# You can also revert all proposed changes with: garage layout revert

# 3. 正式提交生效第1版 layout
garage layout apply --version 1

# ==== COMPUTATION OF A NEW PARTITION ASSIGNATION ====

# Partitions are replicated 1 times on at least 1 distinct zones.

# Optimal partition size: 37.3 MiB
# Usable capacity / total cluster capacity: 9.3 GiB / 9.3 GiB (100.0 %)
# Effective capacity (replication factor 1): 9.3 GiB

# dc1 Tags Partitions Capacity Usable capacity
# 5c81343aa6d99fde [] 256 (256 new) 9.3 GiB 9.3 GiB (100.0%)
# TOTAL 256 (256 unique) 9.3 GiB 9.3 GiB (100.0%)


# New cluster layout with updated role assignment has been applied in cluster.
# Data will now be moved around between nodes accordingly.

# 4. 再次检查状态,确认角色、Zone、Capacity 已生效
garage status

# ==== HEALTHY NODES ====
# ID Hostname Address Tags Zone Capacity DataAvail ResyncQueue Version
# 5c81343aa6d99fde DESKTOP-97FP0NF 127.0.0.1:3901 [] dc1 9.3 GiB 902.3 GiB (89.6%) 0 v2.5.0

创建存储桶跟访问凭证

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
# 1. 创建key,用于管理需要的bucket
garage key create myapp-dev-s3-key

# ==== ACCESS KEY INFORMATION ====
# Key ID: GKeafdc50fbd87ccbf1c2d6785
# Key name: myapp-dev-s3-key
# Secret key: c1f92a47d29336d0a73b38cb46848c22bdd85d39588bf59975fc2b9b2884561d
# Created: 2026-11-20 11:32:59.338 +08:00
# Validity: valid
# Expiration: never

# Can create buckets: false

# ==== BUCKETS FOR THIS KEY ====
# Permissions ID Global aliases Local aliases

# 2. 创建bucket
garage bucket create myapp-public-dev

# ==== BUCKET INFORMATION ====
# Bucket: 4e91cd11b498980ab52b8dafb20289077b05b14205e5905be73413d1751c5859
# Created: 2026-11-20 11:27:37.236 +08:00

# Size: 0 B (0 B)
# Objects: 0

# Website access: false

# Global alias: myapp-public-dev

# ==== KEYS FOR THIS BUCKET ====
# Permissions Access key Local aliases

# 3. 授予权限
garage bucket allow --read --write --owner myapp-public-dev --key myapp-dev-s3-key

# ==== BUCKET INFORMATION ====
# Bucket: 4e91cd11b498980ab52b8dafb20289077b05b14205e5905be73413d1751c5859
# Created: 2026-11-20 11:27:37.236 +08:00

# Size: 0 B (0 B)
# Objects: 0

# Website access: false

# Global alias: myapp-public-dev

# ==== KEYS FOR THIS BUCKET ====
# Permissions Access key Local aliases
# RWO GKeafdc50fbd87ccbf1c2d6785 myapp-dev-s3-key

# 授权直接访问
garage bucket website --allow myapp-public-dev

# ==== BUCKET INFORMATION ====
# Bucket: 4e91cd11b498980ab52b8dafb20289077b05b14205e5905be73413d1751c5859
# Created: 2026-11-20 11:27:37.236 +08:00

# Size: 0 B (0 B)
# Objects: 0

# Website access: true
# index document: index.html
# error document: (not defined)

# Global alias: myapp-public-dev

# ==== KEYS FOR THIS BUCKET ====
# Permissions Access key Local aliases
# RWO GKeafdc50fbd87ccbf1c2d6785 myapp-dev-s3-key

# 结果同上
garage bucket info myapp-public-dev

用s3工具验证

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
# 填入myapp-dev-s3-key的Key ID跟Secret key
export AWS_ACCESS_KEY_ID="GKeafdc50fbd87ccbf1c2d6785"
export AWS_SECRET_ACCESS_KEY="c1f92a47d29336d0a73b38cb46848c22bdd85d39588bf59975fc2b9b2884561d"
export AWS_DEFAULT_REGION="garage"

echo "hello garage cluster" > /tmp/hello.txt

# 上传
aws --endpoint-url http://127.0.0.1:3900 s3 cp /tmp/hello.txt s3://myapp-public-dev/module-test/2026/11/u000001/6c98f490d246c63d7de26fec6ab2311d.txt

# 下载
aws --endpoint-url http://127.0.0.1:3900 s3 cp s3://myapp-public-dev/module-test/2026/11/u000001/6c98f490d246c63d7de26fec6ab2311d.txt /tmp/hello-down.txt

# 对比验证
cat /tmp/hello-down.txt

# 需要开启s3_web
curl -i http://127.0.0.1:3902/myapp-public-dev/module-test/2026/11/u000001/6c98f490d246c63d7de26fec6ab2311d.txt

Nginx开放bucket

编辑nginx配置文件vi /etc/nginx/conf.d/myapp-public-dev.cdn.mydomain.com

1
2
3
4
5
6
7
8
9
10
11
12
13
14
server {
listen 80;
server_name myapp-public-dev.cdn.mydomain.com;

location / {
proxy_pass http://127.0.0.1:3902;

proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

proxy_http_version 1.1;
}
}

测试

1
2
3
4
5
6
7
8
9
10
11
12
13
curl -i http://myapp-public-dev.cdn.mydomain.com/module-test/2026/11/u000001/6c98f490d246c63d7de26fec6ab2311d.txt

# HTTP/1.1 200 OK
# Server: nginx/1.24.0 (Ubuntu)
# Date: Fri, 20 Nov 2026 06:38:34 GMT
# Content-Type: text/plain
# Content-Length: 21
# Connection: keep-alive
# last-modified: Fri, 20 Nov 2026 03:52:10 GMT
# accept-ranges: bytes
# etag: "23502dee82c1244624b0fb1b99bf98ba"

# hello garage cluster