feat 初始化

db/kafka/client.go

@@ -44,7 +44,7 @@ func getConsumer(cfg *config.KafkaConfig, appName string) (sarama.ConsumerGroup,
 	conf := sarama.NewConfig()
 	conf.Consumer.Return.Errors = true
 	conf.Consumer.Group.Session.Timeout = 10 * time.Second
-	conf.Consumer.Offsets.AutoCommit.Enable = false
+	conf.Consumer.Offsets.AutoCommit.Enable = true
 	conf.Consumer.Offsets.Initial = sarama.OffsetOldest
 	return sarama.NewConsumerGroup(cfg.Brokers, appName, conf)
 }

db/kafka/consumer.go

@@ -12,6 +12,8 @@ import (
 	"go.opentelemetry.io/otel/attribute"
 	otelcodes "go.opentelemetry.io/otel/codes"
 	"go.opentelemetry.io/otel/trace"
+	"reflect"
+	"sync"
 	"time"
 )
 
@@ -31,10 +33,13 @@ type Consumer struct {
 func (c *Consumer) Consume(t []Topic) {
 	for {
 		topicArr := make([]string, 0)
-		handlerMap := make(map[string]Topic)
+		handlerMap := make(map[string]*sync.Pool)
 		for _, v := range t {
 			topicArr = append(topicArr, v.Name())
-			handlerMap[v.Name()] = v
+			typ := reflect.ValueOf(v).Elem().Type()
+			handlerMap[v.Name()] = &sync.Pool{New: func() interface{} {
+				return reflect.New(typ).Interface()
+			}}
 		}
 		err := client.consumer.Consume(c.ctx, topicArr, &handler{
 			handler: handlerMap,
@@ -48,13 +53,13 @@ func (c *Consumer) Consume(t []Topic) {
 
 func (c *Consumer) Stop() error {
 	c.cancel()
-	return nil
+	return client.consumer.Close()
 }
 
 type Handler func(context.Context, []byte) error
 
 type handler struct {
-	handler map[string]Topic
+	handler map[string]*sync.Pool
 }
 
 func (h *handler) Setup(_ sarama.ConsumerGroupSession) error {
@@ -67,39 +72,72 @@ func (h *handler) Cleanup(sess sarama.ConsumerGroupSession) error {
 }
 
 func (h *handler) ConsumeClaim(sess sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error {
-	for message := range claim.Messages() {
-		ctx := NewCarrier().ExtractConsumer(message.Headers)
-		_, span := otel.Tracer("common.db.kafka").Start(ctx, "kafka.consume")
+	for {
+		select {
+		case message, ok := <-claim.Messages():
+			if ok {
+				ctx := NewCarrier().ExtractConsumer(message.Headers)
+				_, span := otel.Tracer("common.db.kafka").Start(ctx, "kafka.consume")
 
-		cb := h.handler[message.Topic]
-		if cb == nil {
-			span.SetStatus(otelcodes.Error, "handler not found")
-			span.End()
-			return utils.ErrorsWrap(errors.New("handler not found"))
-		}
+				pool := h.handler[message.Topic]
+				if pool == nil {
+					span.SetStatus(otelcodes.Error, "handler not found")
+					span.End()
+					return utils.ErrorsWrap(errors.New("handler not found"))
+				}
+				span.SetAttributes(attribute.String("kafka.topic", message.Topic))
+				span.SetAttributes(attribute.String("kafka.message", string(message.Value)))
 
-		if err := json.Unmarshal(message.Value, cb); err != nil {
-			span.SetStatus(otelcodes.Error, "handler json.Unmarshal error")
-			span.End()
-			return utils.ErrorsWrap(err, "handler json.Unmarshal error")
-		}
+				newCb := pool.Get().(Topic)
+				if err := json.Unmarshal(message.Value, newCb); err != nil {
+					releaseTopicObject(pool, newCb)
+					span.SetStatus(otelcodes.Error, "handler json.Unmarshal error")
+					span.End()
+					return utils.ErrorsWrap(err, "handler json.Unmarshal error")
+				}
 
-		if err := cb.OnMessage(ctx); err != nil {
-			if stack, ok := err.(interface{ StackTrace() string }); ok {
-				span.AddEvent("Stack Trace", trace.WithAttributes(
-					attribute.String("stack.trace", fmt.Sprintf("%v", stack.StackTrace())),
-				))
+				if err := func() (err error) {
+					defer func() {
+						if e := recover(); e != nil {
+							span.SetStatus(otelcodes.Error, fmt.Sprintf("%v", e))
+							span.End()
+							err = fmt.Errorf("kafka handler panic: %v", e)
+						}
+					}()
+					if err = newCb.OnMessage(ctx); err != nil {
+						if stack, ok := err.(interface{ StackTrace() string }); ok {
+							span.AddEvent("Stack Trace", trace.WithAttributes(
+								attribute.String("stack.trace", fmt.Sprintf("%v", stack.StackTrace())),
+							))
+						}
+						span.SetStatus(otelcodes.Error, err.Error())
+						span.End()
+						log.Errorf("kafka handler error: %v", err)
+						err = utils.ErrorsWrap(err, "kafka handler error")
+					}
+					return
+				}(); err != nil {
+					releaseTopicObject(pool, newCb)
+					return err
+				}
+
+				sess.MarkMessage(message, "")
+				releaseTopicObject(pool, newCb)
+				span.End()
 			}
-			span.SetStatus(otelcodes.Error, err.Error())
-			span.End()
-			return utils.ErrorsWrap(err, "kafka handler error")
+		case <-sess.Context().Done():
+			log.Warnf("kafka session done, reason: %v", sess.Context().Err())
+			sess.Commit()
+			return sess.Context().Err()
 		}
-
-		sess.MarkMessage(message, "")
-		span.End()
 	}
-	sess.Commit()
-	return nil
+}
+
+// 回收Topic对象
+func releaseTopicObject(pool *sync.Pool, obj Topic) {
+	v := reflect.ValueOf(obj).Elem()
+	v.Set(reflect.Zero(v.Type()))
+	pool.Put(obj)
 }
 
 func consumerError() {

db/kafka/producer.go

@@ -3,26 +3,39 @@ package kafka
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"github.com/IBM/sarama"
 	"go.opentelemetry.io/otel"
 	otelcodes "go.opentelemetry.io/otel/codes"
+	"strconv"
 )
 
-func NewProducer() *Producer {
-	return &Producer{}
+var ProduceTimeoutError = errors.New("kafka producer timeout")
+
+func NewProducer(key int64) *Producer {
+	return &Producer{
+		key: key,
+	}
 }
 
 type Producer struct {
+	key int64
 }
 
-func (c *Producer) Produce(ctx context.Context, data Topic) {
+func (c *Producer) Produce(ctx context.Context, data Topic) error {
 	marshal, _ := json.Marshal(data)
 	msg := &sarama.ProducerMessage{
 		Topic:   data.Name(),
+		Key:     sarama.StringEncoder(strconv.FormatInt(c.key, 10)),
 		Value:   sarama.ByteEncoder(marshal),
 		Headers: NewCarrier().Inject(ctx),
 	}
-	client.producer.Input() <- msg
+	select {
+	case client.producer.Input() <- msg:
+		return nil
+	case <-ctx.Done():
+		return ProduceTimeoutError
+	}
 }
 
 func producerError() {

db/kafka/topic.go

@@ -6,43 +6,3 @@ type Topic interface {
 	Name() string
 	OnMessage(context.Context) error
 }
-
-type TopicQuestionAnswer struct {
-	MessageSn      int64  `json:"messageSn"`
-	USN            int64  `json:"usn"`
-	QuestionSn     int64  `json:"questionSn"`
-	QuestionAnswer string `json:"questionAnswer"`
-	Answer         string `json:"answer"`
-}
-
-func (t *TopicQuestionAnswer) Name() string {
-	return "qgdzs.question.answer"
-}
-
-func (t *TopicQuestionAnswer) OnMessage(_ context.Context) error {
-	return nil
-}
-
-type TopicAddPoint struct {
-	MessageSn int64          `json:"messageSn"`
-	Source    AddPointSource `json:"source"`
-	USN       int64          `json:"usn"`
-	Point     int64          `json:"point"`
-}
-
-type AddPointSource int32
-
-const (
-	AddPointSourceUnKnown  AddPointSource = 0
-	AddPointSourceRandom   AddPointSource = 1
-	AddPointSourceCategory AddPointSource = 2
-	AddPointSourceQuickly  AddPointSource = 3
-)
-
-func (t *TopicAddPoint) Name() string {
-	return "qgdzs.user.point"
-}
-
-func (t *TopicAddPoint) OnMessage(_ context.Context) error {
-	return nil
-}

db/mongo/client.go

@@ -4,8 +4,17 @@ import (
 	"context"
 	"git.hlsq.asia/alex/service-common/config"
 	"git.hlsq.asia/alex/service-common/log"
+	"go.mongodb.org/mongo-driver/event"
 	"go.mongodb.org/mongo-driver/mongo"
 	"go.mongodb.org/mongo-driver/mongo/options"
+	"go.mongodb.org/mongo-driver/mongo/readpref"
+	"time"
+)
+
+// Colors
+const (
+	Reset       = "\033[0m"
+	BrightGreen = "\033[92m"
 )
 
 type DBName string
@@ -25,12 +34,27 @@ func Init(cfg map[string]*config.MongoConfig) error {
 
 func initOneClient(cfg *config.MongoConfig) (*mongo.Client, error) {
 	opts := options.Client().
-		ApplyURI(cfg.URI)
+		ApplyURI(cfg.URI).
+		SetMonitor(&event.CommandMonitor{
+			Started: func(ctx context.Context, startedEvent *event.CommandStartedEvent) {
+				log.Infof(BrightGreen+"[MongoDB] Started, Command: %v"+Reset, startedEvent.Command.String())
+			},
+			Failed: func(ctx context.Context, failedEvent *event.CommandFailedEvent) {
+				log.Errorf(BrightGreen+"[MongoDB] [%v] Failed, Failure: %v"+Reset, failedEvent.Duration, failedEvent.Failure)
+			},
+		})
 
-	client, err := mongo.Connect(context.Background(), opts)
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	client, err := mongo.Connect(ctx, opts)
 	if err != nil {
 		return nil, err
 	}
+	if err = client.Ping(ctx, readpref.Primary()); err != nil {
+		_ = client.Disconnect(context.Background())
+		return nil, err
+	}
 
 	return client, nil
 }

db/mysql/client.go

@@ -29,7 +29,7 @@ func initOneDB(cfg *config.MySQLConfig) (*gorm.DB, error) {
 		mysql.Open(cfg.Dsn),
 		&gorm.Config{
 			TranslateError: true,
-			Logger:         logger.Default.LogMode(parseLogLevel(cfg.LogLevel)),
+			Logger:         NewGormLogger(parseLogLevel(cfg.LogLevel)),
 		},
 	)
 	if err != nil {

db/mysql/log.go

@@ -0,0 +1,84 @@
+package mysql
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"git.hlsq.asia/alex/service-common/log"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.uber.org/zap"
+	"gorm.io/gorm"
+	"gorm.io/gorm/logger"
+	"time"
+)
+
+// Colors
+const (
+	Reset = "\033[0m"
+	Blue  = "\033[34m"
+)
+
+type GormLogger struct {
+	logLevel logger.LogLevel
+	logger   *zap.SugaredLogger
+}
+
+func NewGormLogger(level logger.LogLevel) *GormLogger {
+	return &GormLogger{
+		logLevel: level,
+		logger: log.GetLogger().
+			WithOptions(zap.AddCallerSkip(5)).
+			Named("GORM"),
+	}
+}
+
+func (l *GormLogger) LogMode(level logger.LogLevel) logger.Interface {
+	newLogger := *l
+	newLogger.logLevel = level
+	return &newLogger
+}
+
+func (l *GormLogger) Info(ctx context.Context, msg string, data ...interface{}) {
+	if l.logLevel >= logger.Info {
+		l.logger.Infof(Blue+msg+Reset, data...)
+	}
+}
+
+func (l *GormLogger) Warn(ctx context.Context, msg string, data ...interface{}) {
+	if l.logLevel >= logger.Warn {
+		l.logger.Warnf(Blue+msg+Reset, data...)
+	}
+}
+
+func (l *GormLogger) Error(ctx context.Context, msg string, data ...interface{}) {
+	if l.logLevel >= logger.Error {
+		l.logger.Errorf(Blue+msg+Reset, data...)
+	}
+}
+
+func (l *GormLogger) Trace(ctx context.Context, begin time.Time, fc func() (sql string, rowsAffected int64), err error) {
+	if l.logLevel <= logger.Silent {
+		return
+	}
+
+	elapsed := time.Since(begin)
+	sql, rows := fc()
+
+	if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
+		l.logger.Errorf(Blue+"[SLOW SQL] [%v] [rows:%v] [error: %v] %v"+Reset, elapsed, rows, err.Error(), sql)
+		l.SlowLog(ctx, elapsed, sql)
+	} else if elapsed > 50*time.Millisecond {
+		l.logger.Warnf(Blue+"[SLOW SQL] [%v] [rows:%v] %v"+Reset, elapsed, rows, sql)
+		l.SlowLog(ctx, elapsed, sql)
+	} else {
+		l.logger.Infof(Blue+"[%v] [rows:%v] %v"+Reset, elapsed, rows, sql)
+	}
+}
+
+func (l *GormLogger) SlowLog(ctx context.Context, elapsed time.Duration, sql string) {
+	_, span := otel.Tracer("common.db.mysql").Start(ctx, "mysql.slow")
+	defer span.End()
+	span.SetAttributes(attribute.String("elapsed", fmt.Sprintf("%v", elapsed)))
+	span.SetAttributes(attribute.String("sql", sql))
+}

db/redis/cache_client.go

@@ -48,3 +48,11 @@ func (c *CacheClient) Get(ctx context.Context, key string, dst interface{}) bool
 	}
 	return true
 }
+
+func (c *CacheClient) Exists(ctx context.Context, key ...string) bool {
+	n, err := c.cli.Exists(ctx, key...).Result()
+	if err != nil {
+		return false
+	}
+	return n > 0
+}

db/redis/client.go

@@ -5,13 +5,12 @@ import (
 	"git.hlsq.asia/alex/service-common/config"
 	"git.hlsq.asia/alex/service-common/log"
 	"github.com/redis/go-redis/v9"
-	"time"
 )
 
 var instance *Client
 
 type Client struct {
-	cli *redis.Client
+	*redis.Client
 }
 
 func Init(cfg *config.RedisConfig) error {
@@ -21,7 +20,7 @@ func Init(cfg *config.RedisConfig) error {
 		DB:       cfg.DB,
 	})
 	instance = &Client{
-		cli: client,
+		Client: client,
 	}
 	cacheInstance = &CacheClient{
 		cli: client,
@@ -35,8 +34,8 @@ func Init(cfg *config.RedisConfig) error {
 }
 
 func Close() error {
-	if instance != nil && instance.cli != nil {
-		return instance.cli.Close()
+	if instance != nil && instance.Client != nil {
+		return instance.Client.Close()
 	}
 	return nil
 }
@@ -44,43 +43,3 @@ func Close() error {
 func GetClient() *Client {
 	return instance
 }
-
-func (c *Client) Set(ctx context.Context, key string, value interface{}, expiration time.Duration) *redis.StatusCmd {
-	return c.cli.Set(ctx, key, value, expiration)
-}
-
-func (c *Client) SetNX(ctx context.Context, key string, value interface{}, expiration time.Duration) *redis.BoolCmd {
-	return c.cli.SetNX(ctx, key, value, expiration)
-}
-
-func (c *Client) Get(ctx context.Context, key string) *redis.StringCmd {
-	return c.cli.Get(ctx, key)
-}
-
-func (c *Client) HSet(ctx context.Context, key string, values ...interface{}) *redis.IntCmd {
-	return c.cli.HSet(ctx, key, values...)
-}
-
-func (c *Client) HGet(ctx context.Context, key, field string) *redis.StringCmd {
-	return c.cli.HGet(ctx, key, field)
-}
-
-func (c *Client) HGetAll(ctx context.Context, key string) *redis.MapStringStringCmd {
-	return c.cli.HGetAll(ctx, key)
-}
-
-func (c *Client) HDel(ctx context.Context, key string, fields ...string) *redis.IntCmd {
-	return c.cli.HDel(ctx, key, fields...)
-}
-
-func (c *Client) Pipeline() redis.Pipeliner {
-	return c.cli.Pipeline()
-}
-
-func (c *Client) ScriptLoad(ctx context.Context, script string) *redis.StringCmd {
-	return c.cli.ScriptLoad(ctx, script)
-}
-
-func (c *Client) EvalSha(ctx context.Context, sha1 string, keys []string, args ...interface{}) *redis.Cmd {
-	return c.cli.EvalSha(ctx, sha1, keys, args...)
-}