feat: 添加微服务模板基础架构
- 创建基于 CloudWego Hertz 的 Go 微服务脚手架 - 集成 Nacos 服务注册/发现功能 - 添加 gRPC 客户端支持 - 实现环境变量配置管理 (.env.example) - 添加 HTTP 中间件 (Recovery, AccessLog, CORS) - 配置 Gitea CI/CD 构建部署流程 BREAKING CHANGE: 项目结构调整，从简单的 API 服务升级为完整的微服务架构
This commit is contained in:
shiran
@@ -0,0 +1,285 @@
|
||||
package httplog
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/cloudwego/hertz/pkg/app"
|
||||
"github.com/cloudwego/hertz/pkg/common/tracer/stats"
|
||||
"github.com/go-redis/redis/v8"
|
||||
)
|
||||
|
||||
type AccessEvent struct {
|
||||
Timestamp string `json:"@timestamp"`
|
||||
TsMs int64 `json:"ts_ms"`
|
||||
Service string `json:"service,omitempty"`
|
||||
Instance string `json:"instance,omitempty"`
|
||||
Method string `json:"method"`
|
||||
Host string `json:"host,omitempty"`
|
||||
Path string `json:"path"`
|
||||
UserID uint `json:"user_id,omitempty"`
|
||||
Body string `json:"body,omitempty"`
|
||||
Query string `json:"query,omitempty"`
|
||||
Result string `json:"result,omitempty"`
|
||||
ResultTruncated bool `json:"result_truncated,omitempty"`
|
||||
RequestURI string `json:"request_uri,omitempty"`
|
||||
Route string `json:"route,omitempty"`
|
||||
Status int `json:"status"`
|
||||
CostMs int64 `json:"cost_ms"`
|
||||
RecvBytes int `json:"recv_bytes"`
|
||||
SendBytes int `json:"send_bytes"`
|
||||
RemoteAddr string `json:"remote_addr,omitempty"`
|
||||
ClientIP string `json:"client_ip,omitempty"`
|
||||
UserAgent string `json:"ua,omitempty"`
|
||||
Referer string `json:"referer,omitempty"`
|
||||
RequestID string `json:"request_id,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Panicked bool `json:"panicked"`
|
||||
PanicValue string `json:"panic_value"`
|
||||
}
|
||||
|
||||
// UserIDExtractor 从请求上下文中提取用户 ID 的函数签名。
|
||||
// 不同项目可根据自身认证方式实现此函数。
|
||||
type UserIDExtractor func(c *app.RequestContext) uint
|
||||
|
||||
type RedisListWriter struct {
|
||||
rdb *redis.Client
|
||||
key string
|
||||
ch chan []byte
|
||||
flushInterval time.Duration
|
||||
maxBatch int
|
||||
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
wg sync.WaitGroup
|
||||
|
||||
dropped uint64
|
||||
}
|
||||
|
||||
func NewRedisListWriter(rdb *redis.Client, key string, queueSize int, maxBatch int, flushInterval time.Duration) *RedisListWriter {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
w := &RedisListWriter{
|
||||
rdb: rdb,
|
||||
key: key,
|
||||
ch: make(chan []byte, queueSize),
|
||||
flushInterval: flushInterval,
|
||||
maxBatch: maxBatch,
|
||||
ctx: ctx,
|
||||
cancel: cancel,
|
||||
}
|
||||
w.wg.Add(1)
|
||||
go w.loop()
|
||||
return w
|
||||
}
|
||||
|
||||
func (w *RedisListWriter) Enqueue(b []byte) {
|
||||
select {
|
||||
case w.ch <- b:
|
||||
default:
|
||||
atomic.AddUint64(&w.dropped, 1)
|
||||
}
|
||||
}
|
||||
|
||||
func (w *RedisListWriter) Dropped() uint64 { return atomic.LoadUint64(&w.dropped) }
|
||||
|
||||
func (w *RedisListWriter) Close(ctx context.Context) error {
|
||||
w.cancel()
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
w.wg.Wait()
|
||||
close(done)
|
||||
}()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-done:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (w *RedisListWriter) loop() {
|
||||
defer w.wg.Done()
|
||||
|
||||
ticker := time.NewTicker(w.flushInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
flush := func(buf [][]byte) {
|
||||
if len(buf) == 0 {
|
||||
return
|
||||
}
|
||||
args := make([]interface{}, 0, len(buf))
|
||||
for _, b := range buf {
|
||||
args = append(args, string(b))
|
||||
}
|
||||
cctx, cancel := context.WithTimeout(w.ctx, 2*time.Second)
|
||||
_ = w.rdb.RPush(cctx, w.key, args...).Err()
|
||||
cancel()
|
||||
}
|
||||
|
||||
var buf [][]byte
|
||||
for {
|
||||
select {
|
||||
case <-w.ctx.Done():
|
||||
flush(buf)
|
||||
return
|
||||
case b := <-w.ch:
|
||||
buf = append(buf, b)
|
||||
if len(buf) >= w.maxBatch {
|
||||
flush(buf)
|
||||
buf = buf[:0]
|
||||
}
|
||||
case <-ticker.C:
|
||||
flush(buf)
|
||||
buf = buf[:0]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type TracerOption func(*RedisAccessLogTracer)
|
||||
|
||||
// WithUserIDExtractor 设置自定义的用户 ID 提取函数
|
||||
func WithUserIDExtractor(fn UserIDExtractor) TracerOption {
|
||||
return func(t *RedisAccessLogTracer) {
|
||||
t.userIDFn = fn
|
||||
}
|
||||
}
|
||||
|
||||
// WithSkipPrefix 设置需要跳过日志记录的路径前缀（如 /health, /metrics）
|
||||
func WithSkipPrefix(prefix string) TracerOption {
|
||||
return func(t *RedisAccessLogTracer) {
|
||||
t.skipPrefix = prefix
|
||||
}
|
||||
}
|
||||
|
||||
// WithMaxResponseBody 设置响应体采集的最大字节数
|
||||
func WithMaxResponseBody(n int) TracerOption {
|
||||
return func(t *RedisAccessLogTracer) {
|
||||
t.maxRespBody = n
|
||||
}
|
||||
}
|
||||
|
||||
type RedisAccessLogTracer struct {
|
||||
writer *RedisListWriter
|
||||
service string
|
||||
instance string
|
||||
skipPrefix string
|
||||
maxRespBody int
|
||||
userIDFn UserIDExtractor
|
||||
}
|
||||
|
||||
func NewRedisAccessLogTracer(rdb *redis.Client, redisKey string, service string, opts ...TracerOption) *RedisAccessLogTracer {
|
||||
host, _ := os.Hostname()
|
||||
t := &RedisAccessLogTracer{
|
||||
writer: NewRedisListWriter(rdb, redisKey, 8192, 512, 200*time.Millisecond),
|
||||
service: service,
|
||||
instance: host,
|
||||
maxRespBody: 2048,
|
||||
}
|
||||
for _, opt := range opts {
|
||||
opt(t)
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
func (t *RedisAccessLogTracer) Start(ctx context.Context, _ *app.RequestContext) context.Context {
|
||||
return ctx
|
||||
}
|
||||
|
||||
func (t *RedisAccessLogTracer) Finish(_ context.Context, c *app.RequestContext) {
|
||||
if t.skipPrefix != "" {
|
||||
p := string(c.Request.URI().PathOriginal())
|
||||
if len(p) >= len(t.skipPrefix) && p[:len(t.skipPrefix)] == t.skipPrefix {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
ti := c.GetTraceInfo()
|
||||
st := ti.Stats()
|
||||
|
||||
var cost time.Duration
|
||||
if rpcStart := st.GetEvent(stats.HTTPStart); rpcStart != nil {
|
||||
if rpcFinish := st.GetEvent(stats.HTTPFinish); rpcFinish != nil {
|
||||
cost = rpcFinish.Time().Sub(rpcStart.Time())
|
||||
}
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
req := &c.Request
|
||||
uri := req.URI()
|
||||
|
||||
remoteAddr := ""
|
||||
if ra := c.RemoteAddr(); ra != nil {
|
||||
remoteAddr = ra.String()
|
||||
}
|
||||
|
||||
errStr := ""
|
||||
if st.Error() != nil {
|
||||
errStr = st.Error().Error()
|
||||
}
|
||||
panicked, panicVal := st.Panicked()
|
||||
panicStr := ""
|
||||
if panicVal != nil {
|
||||
panicStr = anyToString(panicVal)
|
||||
}
|
||||
|
||||
ev := AccessEvent{
|
||||
Timestamp: now.Format(time.RFC3339Nano),
|
||||
TsMs: now.UnixMilli(),
|
||||
Service: t.service,
|
||||
Instance: t.instance,
|
||||
Method: string(req.Method()),
|
||||
Host: string(req.Host()),
|
||||
Path: string(uri.PathOriginal()),
|
||||
Query: string(uri.QueryString()),
|
||||
RequestURI: string(uri.RequestURI()),
|
||||
Status: c.Response.StatusCode(),
|
||||
CostMs: cost.Milliseconds(),
|
||||
RecvBytes: st.RecvSize(),
|
||||
SendBytes: st.SendSize(),
|
||||
RemoteAddr: remoteAddr,
|
||||
ClientIP: c.ClientIP(),
|
||||
UserAgent: string(req.Header.UserAgent()),
|
||||
Referer: string(req.Header.Peek("Referer")),
|
||||
RequestID: string(req.Header.Peek("X-Request-Id")),
|
||||
Error: errStr,
|
||||
Panicked: panicked,
|
||||
PanicValue: panicStr,
|
||||
}
|
||||
|
||||
if t.userIDFn != nil {
|
||||
ev.UserID = t.userIDFn(c)
|
||||
}
|
||||
|
||||
if formJSON, ok, _ := FormBodyToJSONWithFilePlaceholder(c); ok {
|
||||
ev.Body = string(formJSON)
|
||||
}
|
||||
|
||||
if respBody, ok, trunc := ResponseBodySnippet(c, t.maxRespBody); ok {
|
||||
ev.Result = respBody
|
||||
ev.ResultTruncated = trunc
|
||||
}
|
||||
|
||||
b, err := json.Marshal(ev)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
t.writer.Enqueue(b)
|
||||
}
|
||||
|
||||
func (t *RedisAccessLogTracer) Close(ctx context.Context) error {
|
||||
return t.writer.Close(ctx)
|
||||
}
|
||||
|
||||
func anyToString(v interface{}) string {
|
||||
switch x := v.(type) {
|
||||
case string:
|
||||
return x
|
||||
default:
|
||||
b, _ := json.Marshal(x)
|
||||
return string(b)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
package httplog
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"mime"
|
||||
"strings"
|
||||
|
||||
"github.com/cloudwego/hertz/pkg/app"
|
||||
)
|
||||
|
||||
var sensitiveKeys = []string{"password", "secret", "token", "key", "passwd"}
|
||||
|
||||
const redactedValue = "[REDACTED]"
|
||||
|
||||
// desensitization：只对第一层 key 做脱敏，不递归
|
||||
func desensitization(data map[string]any) ([]byte, error) {
|
||||
out := make(map[string]any, len(data))
|
||||
|
||||
for k, v := range data {
|
||||
if isSensitiveKey(k) {
|
||||
out[k] = redactedValue
|
||||
} else {
|
||||
out[k] = v
|
||||
}
|
||||
}
|
||||
return json.Marshal(out)
|
||||
}
|
||||
|
||||
// 大小写不敏感、包含式匹配
|
||||
func isSensitiveKey(key string) bool {
|
||||
k := strings.ToLower(key)
|
||||
for _, s := range sensitiveKeys {
|
||||
if strings.Contains(k, s) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// FormBodyToJSONWithFilePlaceholder ：
|
||||
// - 普通表单：完整转 JSON
|
||||
// - multipart 文件字段：不取任何文件信息，只输出 "[file]" 或 ["[file]", ...]
|
||||
func FormBodyToJSONWithFilePlaceholder(c *app.RequestContext) (jsonBytes []byte, ok bool, err error) {
|
||||
ct := string(c.Request.Header.ContentType())
|
||||
mediaType, _, _ := mime.ParseMediaType(ct)
|
||||
mediaType = strings.ToLower(mediaType)
|
||||
|
||||
out := make(map[string]any)
|
||||
|
||||
switch mediaType {
|
||||
case "application/x-www-form-urlencoded":
|
||||
args := c.PostArgs()
|
||||
args.VisitAll(func(k, v []byte) {
|
||||
key := string(k)
|
||||
// 同名 key 多次出现时，转成数组
|
||||
if old, exists := out[key]; exists {
|
||||
switch x := old.(type) {
|
||||
case string:
|
||||
out[key] = []string{x, string(v)}
|
||||
case []string:
|
||||
out[key] = append(x, string(v))
|
||||
default:
|
||||
out[key] = string(v)
|
||||
}
|
||||
} else {
|
||||
out[key] = string(v)
|
||||
}
|
||||
})
|
||||
b, e := desensitization(out)
|
||||
return b, true, e
|
||||
|
||||
case "multipart/form-data":
|
||||
form, e := c.MultipartForm()
|
||||
if e != nil {
|
||||
return nil, false, e
|
||||
}
|
||||
|
||||
// 普通字段
|
||||
for k, vv := range form.Value {
|
||||
if len(vv) == 1 {
|
||||
out[k] = vv[0]
|
||||
} else {
|
||||
out[k] = vv
|
||||
}
|
||||
}
|
||||
|
||||
// 文件字段：只做占位，不读取任何文件信息
|
||||
for k, files := range form.File {
|
||||
if len(files) <= 1 {
|
||||
out[k] = "[file]"
|
||||
} else {
|
||||
arr := make([]string, 0, len(files))
|
||||
for range files {
|
||||
arr = append(arr, "[file]")
|
||||
}
|
||||
out[k] = arr
|
||||
}
|
||||
}
|
||||
|
||||
b, e2 := desensitization(out)
|
||||
return b, true, e2
|
||||
|
||||
default:
|
||||
// 不是表单就不处理
|
||||
return nil, false, nil
|
||||
}
|
||||
}
|
||||
|
||||
// ResponseBodySnippet ：获取响应内容
|
||||
func ResponseBodySnippet(c *app.RequestContext, maxBytes int) (body string, ok bool, truncated bool) {
|
||||
b := c.Response.Body()
|
||||
if len(b) == 0 {
|
||||
return "", false, false
|
||||
}
|
||||
|
||||
// 只采集“看起来像文本/JSON”的响应，其他（图片/zip等）直接跳过
|
||||
ct := strings.ToLower(string(c.Response.Header.ContentType()))
|
||||
if ct != "" && !(strings.HasPrefix(ct, "text/") ||
|
||||
strings.Contains(ct, "application/json") ||
|
||||
strings.Contains(ct, "application/xml") ||
|
||||
strings.Contains(ct, "application/javascript")) {
|
||||
return "", false, false
|
||||
}
|
||||
|
||||
if maxBytes > 0 && len(b) > maxBytes {
|
||||
b = b[:maxBytes]
|
||||
truncated = true
|
||||
}
|
||||
|
||||
body = string(b)
|
||||
return body, true, truncated
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
package httplog
|
||||
|
||||
import (
|
||||
"apiServer_service/utils/logger"
|
||||
"apiServer_service/utils/redis_tools"
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/go-redis/redis/v8"
|
||||
)
|
||||
|
||||
type AccessEventData struct {
|
||||
Timestamp string `json:"@timestamp"`
|
||||
TsMs int64 `json:"ts_ms"`
|
||||
}
|
||||
|
||||
var popBatchLua = redis.NewScript(`
|
||||
local key = KEYS[1]
|
||||
local n = tonumber(ARGV[1])
|
||||
local res = redis.call("LRANGE", key, 0, n-1)
|
||||
if (#res > 0) then
|
||||
redis.call("LTRIM", key, n, -1)
|
||||
end
|
||||
return res
|
||||
`)
|
||||
|
||||
func getenv(key, def string) string {
|
||||
if v := os.Getenv(key); v != "" {
|
||||
return v
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
func getenvInt(key string, def int) int {
|
||||
v := os.Getenv(key)
|
||||
if v == "" {
|
||||
return def
|
||||
}
|
||||
i, err := strconv.Atoi(v)
|
||||
if err != nil {
|
||||
return def
|
||||
}
|
||||
return i
|
||||
}
|
||||
|
||||
func Updater(redisKey, esIndexPrefix string) {
|
||||
if redisKey == "" {
|
||||
redisKey = getenv("ES_REDIS_KEY", "access_log")
|
||||
}
|
||||
|
||||
esBulkURL := getenv("ES_BULK_URL", "https://elasticsearch.hostidc.net/_bulk")
|
||||
if esIndexPrefix == "" {
|
||||
esIndexPrefix = getenv("ES_INDEX_PREFIX", "access")
|
||||
}
|
||||
esUser := getenv("ES_USERNAME", "")
|
||||
esPass := getenv("ES_PASSWORD", "")
|
||||
|
||||
batchSize := getenvInt("ES_BATCH_SIZE", 1000)
|
||||
pollIntervalMs := getenvInt("ES_POLL_INTERVAL_MS", 200)
|
||||
httpTimeoutMs := getenvInt("ES_HTTP_TIMEOUT_MS", 5000)
|
||||
|
||||
rdb := redis_tools.ConnectRedis()
|
||||
|
||||
httpClient := &http.Client{Timeout: time.Duration(httpTimeoutMs) * time.Millisecond}
|
||||
ctx := context.Background()
|
||||
|
||||
for {
|
||||
items, err := popBatchLua.Run(ctx, rdb, []string{redisKey}, batchSize).StringSlice()
|
||||
if err != nil {
|
||||
time.Sleep(time.Second)
|
||||
continue
|
||||
}
|
||||
if len(items) == 0 {
|
||||
time.Sleep(time.Duration(pollIntervalMs) * time.Millisecond)
|
||||
continue
|
||||
}
|
||||
|
||||
body, _ := buildBulkBody(items, esIndexPrefix)
|
||||
ok, _, err := postBulk(ctx, httpClient, esBulkURL, esUser, esPass, body)
|
||||
if err != nil || !ok {
|
||||
logger.Error("ESBulk", "写入失败: ", err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func buildBulkBody(items []string, indexPrefix string) ([]byte, int) {
|
||||
var buf bytes.Buffer
|
||||
count := 0
|
||||
|
||||
for _, line := range items {
|
||||
idx := indexPrefix + "-" + time.Now().Format("2006.01.02")
|
||||
|
||||
var ev AccessEventData
|
||||
if json.Unmarshal([]byte(line), &ev) == nil {
|
||||
if ev.TsMs > 0 {
|
||||
idx = indexPrefix + "-" + time.UnixMilli(ev.TsMs).UTC().Format("2006.01.02")
|
||||
} else if ev.Timestamp != "" {
|
||||
if ts, err := time.Parse(time.RFC3339Nano, ev.Timestamp); err == nil {
|
||||
idx = indexPrefix + "-" + ts.UTC().Format("2006.01.02")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sum := sha1.Sum([]byte(line))
|
||||
docID := hex.EncodeToString(sum[:])
|
||||
|
||||
meta := fmt.Sprintf(`{"index":{"_index":"%s","_id":"%s"}}`+"\n", idx, docID)
|
||||
buf.WriteString(meta)
|
||||
buf.WriteString(line)
|
||||
buf.WriteByte('\n')
|
||||
count++
|
||||
}
|
||||
return buf.Bytes(), count
|
||||
}
|
||||
|
||||
func postBulk(ctx context.Context, client *http.Client, url, user, pass string, body []byte) (bool, []byte, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return false, nil, err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/x-ndjson")
|
||||
if user != "" || pass != "" {
|
||||
req.SetBasicAuth(user, pass)
|
||||
}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return false, nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
return false, b, fmt.Errorf("bulk http status=%d", resp.StatusCode)
|
||||
}
|
||||
|
||||
var out struct {
|
||||
Errors bool `json:"errors"`
|
||||
}
|
||||
if json.Unmarshal(b, &out) == nil && out.Errors {
|
||||
return false, b, fmt.Errorf("bulk response errors=true")
|
||||
}
|
||||
|
||||
return true, b, nil
|
||||
}
|
||||
Reference in New Issue
Block a user