一、Rustfs部署（使用nfs作为持久化存储）
- 创建Rustfs-statefulset.yaml文件
apiVersion: v1
kind: Service
metadata:
name: rustfs
namespace: default
spec:
type: NodePort
selector:
app: rustfs
ports:
- name: rustfs
port: 9000
targetPort: 9000
- name: console
port: 9001
targetPort: 9001
---
apiVersion: v1
kind: ConfigMap
metadata:
name: rustfs-config
namespace: default
data:
start-rustfs.sh: |
#!/bin/sh
set -e
POD_NAME=${POD_NAME}
REPLICAS=${RUSTFS_REPLICAS}
RUSTFS_VOLUMES="http://rustfs-{0...$((REPLICAS-1))}.rustfs:9000/data"
sleep 10
exec rustfs $RUSTFS_VOLUMES
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
labels:
app: rustfs
name: rustfs
namespace: default
spec:
serviceName: rustfs
replicas: 4 # 必须与环境变量RUSTFS_REPLICAS值一致
selector:
matchLabels:
app: rustfs
template:
metadata:
labels:
app: rustfs
spec:
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchExpressions:
- key: app
operator: In
values:
- rustfs
topologyKey: kubernetes.io/hostname
containers:
- name: rustfs
image: ccr.ccs.tencentyun.com/zoehuawang/rustfs:1.0.0-alpha.89
command: ["/bin/sh","/scripts/start-rustfs.sh"]
env:
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: RUSTFS_REPLICAS # 必须与上面spec.replicas值一致
value: "4"
- name: RUSTFS_ACCESS_KEY
value: "admin"
- name: RUSTFS_SECRET_KEY
value: "Aa123456"
- name: RUSTFS_CONSOLE_ENABLE
value: "true"
- name: RUSTFS_ADDRESS
value: ":9000"
- name: RUSTFS_CONSOLE_ADDRESS
value: ":9001"
- name: RUST_LOG
value: "error"
ports:
- containerPort: 9000
name: rustfs
- containerPort: 9001
name: console
resources:
requests:
memory: "128Mi"
cpu: "100m"
limits:
memory: "4Gi"
cpu: "2000m"
securityContext:
runAsUser: 0
runAsGroup: 0
volumeMounts:
- mountPath: /scripts
name: startup-scripts
- mountPath: /data
name: rustfs-storage
subPathExpr: $(POD_NAME)
volumes:
- name: startup-scripts
configMap:
name: rustfs-config
defaultMode: 0755
- name: rustfs-storage
nfs:
server: 172.16.200.53
path: /data/rustfs
readOnly: false
#hostPath:
# path: /data/rustfs
# type: DirectoryOrCreate
- 在172.16.200.53的nfs共享目录下创建数据目录
mkdir -p /data/rustfs
- 启动rustfs
kubectl apply -f rustfs-satefulset.yaml
- 通过用户admin，密码Aa123456，访问rustfs的webui页面(9001端口对应的nodeport端口)
- 创建存储桶，名称与Loki配置的bucketnames一致
二、Loki部署
- 创建loki-config.yaml文件，注意yaml文件中注释自行修改
apiVersion: v1
kind: ConfigMap
metadata:
name: loki-config
namespace: default
data:
local-config.yaml: |
auth_enabled: false
server:
http_listen_port: 3100
grpc_listen_port: 9096
log_level: info
grpc_server_max_recv_msg_size: 10485760
grpc_server_max_send_msg_size: 10485760
common:
path_prefix: /loki
storage:
s3:
endpoint: rustfs:9000
insecure: true
bucketnames: loki-logs1,loki-logs2,loki-logs3,loki-logs4 # 需要提前在rustfs页面上提前创建好
access_key_id: admin # rustfs的yaml中配置的用户名
secret_access_key: Aa123456 # rustfs的yaml中配置的密码
s3forcepathstyle: true
replication_factor: 1
ring:
kvstore:
store: memberlist
storage_config:
tsdb_shipper:
active_index_directory: /loki/tsdb-index
cache_location: /loki/tsdb-cache
cache_ttl: 1h
resync_interval: 20m
compactor:
working_directory: /loki/compactor
compaction_interval: 2h # 执行清理间隔时间
retention_enabled: true # 启用清理
retention_delete_delay: 1h # 延迟1h清理
retention_delete_worker_count: 3
delete_request_store: s3
compactor_ring:
kvstore:
store: memberlist
memberlist:
bind_addr: ['0.0.0.0']
bind_port: 7946
join_members:
- "loki.default.svc.cluster.local"
dead_node_reclaim_time: 30s
gossip_to_dead_nodes_time: 15s
left_ingesters_timeout: 30s
node_name: ${HOSTNAME}
randomize_node_name: false
abort_if_cluster_join_fails: false
ingester:
chunk_encoding: snappy # 日志块在存储前的压缩编码方式
max_chunk_age: 30m # 日志块在存储前的的最大时间
chunk_target_size: 3145728 # 日志块在存储前的的目标大小,3MB
chunk_idle_period: 2m # 日志块在存储前的空闲时间
chunk_retain_period: 30s # 日志块关闭后在内存的保留时间
lifecycler:
ring:
kvstore:
store: memberlist
replication_factor: 1
wal:
enabled: true
dir: /loki/wal
limits_config:
max_line_size: 1048576
split_queries_by_interval: 30m
query_timeout: 300s
metric_aggregation_enabled: true
enable_multi_variant_queries: true
max_entries_limit_per_query: 10000 # 单个查询可以返回的最大日志条目数
retention_period: 168h # 全局日志保留时间,加上延迟时间实际会保留25h
# 建议大于等于索引分片时间period
# 否则得等分片时间结束才会清理
schema_config:
configs:
- from: "2000-01-01"
store: tsdb
object_store: s3
schema: v13
index:
prefix: index_ # 索引文件的前缀名
period: 24h # 索引分片时间,tsdb必须为24h,不能为其他值
- 创建loki-deployment.yaml文件
apiVersion: v1
kind: Service
metadata:
name: loki
namespace: default
labels:
app: loki
spec:
clusterIP: None
type: ClusterIP
ports:
- port: 3100
targetPort: 3100
name: http
- port: 9096
targetPort: 9096
name: grpc
- port: 7946
targetPort: 7946
name: memberlist
selector:
app: loki
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: loki
namespace: default
labels:
app: loki
spec:
replicas: 3
selector:
matchLabels:
app: loki
template:
metadata:
labels:
app: loki
spec:
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchExpressions:
- key: app
operator: In
values:
- loki
topologyKey: kubernetes.io/hostname
imagePullSecrets:
- name: jcr-pull-secret
nodeSelector:
feature: app
containers:
- name: loki
image: harbor-welljoint-cn-east-2.jcr.service.jdcloud.com/loki:3.5.7
imagePullPolicy: IfNotPresent
resources:
limits:
cpu: 2
memory: 4Gi
requests:
cpu: 0.5
memory: 256Mi
env:
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: HOSTNAME
valueFrom:
fieldRef:
fieldPath: metadata.name
args:
- -config.file=/etc/loki/local-config.yaml
- -config.expand-env=true
ports:
- name: http
containerPort: 3100
protocol: TCP
- name: grpc
containerPort: 9096
protocol: TCP
volumeMounts:
- name: loki-config
mountPath: /etc/loki/
- name: host-time
mountPath: /etc/localtime
readOnly: true
volumes:
- name: loki-config
configMap:
name: loki-config
defaultMode: 493
- name: host-time
hostPath:
path: /etc/localtime
type: ""
- 查看Loki集群成员状态
# 使用kubectl将rustfs的pod端口通过本地端口临时代理出来
# 访问页面(http://ip:3100/memberlist)
kubectl port-forward -n default svc/loki 3100:3100 --address 0.0.0.0
三、logstash部署
- 创建logstash-config.yaml文件
```yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: logstash-config
namespace: default
labels:
app: logstash
data:
jvm.options: |+
-Xms1g
-Xmx3g
-XX:+UseG1GC
-XX:MaxGCPauseMillis=200
-XX:G1HeapRegionSize=2m
-XX:InitiatingHeapOccupancyPercent=35
-XX:G1ReservePercent=15
-XX:+ParallelRefProcEnabled
-Djava.awt.headless=true
-Dfile.encoding=UTF-8
-Djruby.compile.invokedynamic=true
-XX:+HeapDumpOnOutOfMemoryError
-Djava.security.egd=file:/dev/urandom
-Dlog4j2.isThreadContextMapInheritable=true
-Dlogstash.jackson.stream-read-constraints.max-string-length=200000000
-Dlogstash.jackson.stream-read-constraints.max-number-length=10000
-Duser.timezone=Asia/Shanghai
logstash.yml: |+
path.config: /usr/share/logstash/config/logstash.conf
pipeline.workers: 2
pipeline.batch.size: 1
pipeline.batch.delay: 0
queue.type: memory
queue.max_events: 200
queue.drain: false
logstash.conf: |+
input{
tcp {
port => 9601
codec => "json"
}
}
filter {
mutate {
remove_field => ["detected_level", "service_name", "@version", "version"]
}
ruby {
code => '
current_time_ns = (Time.now.to_f * 1_000_000_000).to_i.to_s
message = event.get("message") || "unknown"
labels = {
"environment" => event.get("environment") || "UAT&SIT",
"HOSTNAME" => event.get("HOSTNAME") || "unknown",
"threadId" => event.get("threadId") || "",
"serviceId" => event.get("serviceId") || "",
"level" => event.get("level") || ""
}
loki_data = {
"streams" => [
{
"stream" => labels,
"values" => [
[current_time_ns, message]
]
}
]
}.to_json
event.set("[@metadata][loki_data]", loki_data)
'
}
}
output {
http {
url => "http://loki:3100/loki/api/v1/push"
http_method => "post"
content_type => "application/json"
format => "message"
message => "%{[@metadata][loki_data]}"
retryable_codes => [429, 500, 502, 503, 504]
automatic_retries => 2
retry_failed => true
pool_max => 50
pool_max_per_route => 50
keepalive => true
}
}
- 创建logstash-deployment.yaml文件
apiVersion: v1
kind: Service
metadata:
name: logstash
namespace: default
labels:
app: logstash
spec:
selector:
app: logstash
type: NodePort
ports:
- port: 9601
name: logs
nodePort: 30961
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: logstash
namespace: default
spec:
replicas: 3
selector:
matchLabels:
app: logstash
template:
metadata:
labels:
app: logstash
spec:
securityContext:
runAsUser: 0
nodeSelector:
feature: app
containers:
- name: logstash
image: ccr.ccs.tencentyun.com/zoehuawang/logstash:8.14.1
imagePullPolicy: IfNotPresent
resources:
requests:
memory: "500Mi"
cpu: "500m"
limits:
memory: "2Gi"
cpu: "1000m"
ports:
- containerPort: 9601
protocol: TCP
name: logstash
command: ["/bin/sh","-c"]
args:
- |
cat /config/logstash.yml > /usr/share/logstash/config/logstash.yml;
cat /config/logstash.conf > /usr/share/logstash/config/logstash.conf;
cat /config/jvm.options > /usr/share/logstash/config/jvm.options;
/usr/local/bin/docker-entrypoint
env:
- name: LOG_LEVEL
value: "error"
- name: node.name
valueFrom:
fieldRef:
fieldPath: metadata.name
volumeMounts:
- name: logstash-config
mountPath: /config
- name: host-time
mountPath: /etc/localtime
readOnly: true
volumes:
- name: logstash-config
configMap:
name: logstash-config
defaultMode: 420
- name: host-time
hostPath:
path: /etc/localtime
type: ""
四、查询日志方式
- 模拟日志推送
curl -v -X POST http://loki:3100/loki/api/v1/push -H "Content-Type: application/json" -d '{"streams": [{"stream": {"environment": "UAT&SIT"},"values": [["'"$(date +%s)000000000"'","测试推送日志到Loki3"]]}]}'
- 日志关键字查询
curl -G "http://loki:3100/loki/api/v1/query_range" --data-urlencode "query={environment=\"UAT&SIT\"} |= \"测试推送日志到\"" --data-urlencode "start=$(date -d '2025-11-19 15:00:00' +%s)000000000" --data-urlencode "end=$(date -d '2025-11-19 15:30:00' +%s)000000000" --data-urlencode "limit=50"
- 在grafana中查看：
- 1、添加数据源为Loki
- 2、点击Explore,选择对应标签即可
补充一：使用Promtail主动收集日志发送到loki的镜像及用法如下:
# 自定义的配置文件/mnt/config/promtail-config.yaml
docker run -d --name promtail --network=host --restart=always \
-e loki_ip_port="172.23.26.69:3100" \
-v /logs:/var/log/hrsync -v /mnt/logs:/var/log/fs \
ccr.ccs.tencentyun.com/zoehuawang/promtail:3.2.1 \
--config.file=/mnt/config/promtail-config.yaml --config.expand-env=true
补充二：Logstash通过loki插件方式推送日志:
# logstash也有可以直接使用的loki插件，不过好像有内存泄漏问题，后续有修复内存泄漏的插件可以直接使用
# 插件版本地址：https://rubygems.org/gems/logstash-output-loki/versions
# 插件安装：
# 1.在可通互联网的机器上安装好logstash
/usr/share/logstash/bin/logstash-plugin install --version 1.2.0 logstash-output-loki
# 2.通过离线打包的方式，将在线安装好的插件打包，生成插件包（logstash-offline-ouput-8.14.1.zip）
/usr/share/logstash/bin/logstash-plugin prepare-offline-pack logstash-output-loki
# 3.离线安装插件
/usr/share/logstash/bin/logstash-plugin install file:///tmp/logstash-offline-ouput-8.14.1.zip
# 插件使用，配置如下：
logstash.conf: |+
input{
tcp {
port => 9601
codec => "json"
}
}
filter {
mutate {
add_field => { "environment" => "UAT&SIT" }
}
}
output {
loki {
url => "http://loki:3100/loki/api/v1/push"
include_fields => [
"environment",
"HOSTNAME",
"threadId",
"serviceld",
"level"
]
batch_size => 100
batch_wait => 5
retries => 3
}
}