From 473eafc069489dcd0af8c9b342ef78f72bc57a5a Mon Sep 17 00:00:00 2001 From: Havoc412 <2993167370@qq.com> Date: Thu, 7 Nov 2024 21:28:41 +0800 Subject: [PATCH] =?UTF-8?q?=F0=9F=8E=8FFInish=20prefer=20List?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/global/variable/variable.go | 1 + app/http/controller/web/animal_controller.go | 101 ++++++++++++-- app/http/validator/web/animal/list.go | 1 + app/model/animal.go | 6 +- app/model/encounter.go | 46 ++++++- app/service/animals/curd/animals_curd.go | 4 +- app/service/encounter/curd/encounter_curd.go | 5 +- app/utils/redis_factory/client.go | 1 - test/redis_test.go | 135 +++++++++++++++++++ test/snowflake_test.go | 58 ++++++++ 10 files changed, 338 insertions(+), 20 deletions(-) create mode 100644 test/redis_test.go create mode 100644 test/snowflake_test.go diff --git a/app/global/variable/variable.go b/app/global/variable/variable.go index 6d0ae5d..08d8beb 100644 --- a/app/global/variable/variable.go +++ b/app/global/variable/variable.go @@ -12,6 +12,7 @@ import ( "github.com/yankeguo/zhipu" "go.uber.org/zap" "gorm.io/gorm" + ) var ( diff --git a/app/http/controller/web/animal_controller.go b/app/http/controller/web/animal_controller.go index 9bc1fc5..468bb12 100644 --- a/app/http/controller/web/animal_controller.go +++ b/app/http/controller/web/animal_controller.go @@ -8,6 +8,8 @@ import ( "catface/app/model" "catface/app/service/animals/curd" "catface/app/service/upload_file" + "catface/app/utils/query_handler" + "catface/app/utils/redis_factory" "catface/app/utils/response" "os" "path/filepath" @@ -19,6 +21,16 @@ import ( type Animals struct { // INFO 起到一个标记的作用，这样 web.xxx 的时候不同模块就不会命名冲突了。 } +// contains 检查 id 是否在 ids 切片中 +func contains(ids []int, id int) bool { + for _, v := range ids { + if v == id { + return true + } + } + return false +} + func (a *Animals) List(context *gin.Context) { // 1. Get Params attrs := context.GetString(consts.ValidatorPrefix + "attrs") @@ -27,26 +39,95 @@ func (a *Animals) List(context *gin.Context) { sterilization := context.GetString(consts.ValidatorPrefix + "sterilization") status := context.GetString(consts.ValidatorPrefix + "status") department := context.GetString(consts.ValidatorPrefix + "department") - num := context.GetFloat64(consts.ValidatorPrefix + "num") - skip := context.GetFloat64(consts.ValidatorPrefix + "skip") + num := int(context.GetFloat64(consts.ValidatorPrefix + "num")) + skip := int(context.GetFloat64(consts.ValidatorPrefix + "skip")) userId := context.GetFloat64(consts.ValidatorPrefix + "user_id") mode := context.GetString(consts.ValidatorPrefix + "mode") + var preferCatsId []int64 + var redis_num int + var key int64 if mode == consts.AnimalPreferMode { - preferList(context) - } else { // 其余都是 默认模式。 - animals := curd.CreateAnimalsCurdFactory().List(attrs, gender, breed, sterilization, status, department, int(num), int(skip), int(userId)) - if animals != nil { - response.Success(context, consts.CurdStatusOkMsg, animals) + key = int64(context.GetFloat64(consts.ValidatorPrefix + "key")) + + redisClient := redis_factory.GetOneRedisClient() + defer redisClient.ReleaseOneRedisClient() + if key != 0 { + redis_num, _ = redisClient.Int(redisClient.Execute("get", key)) } else { - response.Fail(context, errcode.AnimalNoFind, errcode.ErrMsg[errcode.AnimalNoFind], "") + key = variable.SnowFlake.GetId() } + + if redis_num == skip { + preferCatsId, _ = getPreferCatsId(int(userId), int(num)) + redis_num += len(preferCatsId) + } + + if _, err := redisClient.String(redisClient.Execute("set", key, redis_num)); err != nil { + } + } + + var animalsWithLike []model.AnimalWithLikeList + if len(preferCatsId) > 0 { + // 创建一个 map 来存储查询结果 + animalMap := make(map[int64]model.Animal, len(preferCatsId)) + + attrsSlice := query_handler.StringToStringArray(attrs) + attrsSlice = append(attrsSlice, "id") + animals := model.CreateAnimalFactory("").ShowByIDs(preferCatsId, attrsSlice...) + + for _, v := range animals { + animalMap[v.Id] = v + } + + // 根据 preferCatsId 的顺序构建最终结果列表 + for _, id := range preferCatsId { + if animal, ok := animalMap[id]; ok { + animalsWithLike = append(animalsWithLike, model.AnimalWithLikeList{Animal: animal}) + } + } + } + + // 计算还需要多少动物 + num -= len(animalsWithLike) + skip -= redis_num + if num > 0 { + additionalAnimals := curd.CreateAnimalsCurdFactory().List(attrs, gender, breed, sterilization, status, department, preferCatsId, num, skip, int(userId)) + // 将 additionalAnimals 整合到 animalsWithLike 的后面 + animalsWithLike = append(animalsWithLike, additionalAnimals...) + } + + if animalsWithLike != nil { + response.Success(context, consts.CurdStatusOkMsg, gin.H{ + "animals": animalsWithLike, + "key": key, + }) + } else { + response.Fail(context, errcode.AnimalNoFind, errcode.ErrMsg[errcode.AnimalNoFind], "") } } -func preferList(context *gin.Context) { - // TODO 先去考虑一下前端筛选的实现方式。 +// UPDATE 就先简单一些，主要就依靠 encounter - animal_id 来获取一个目标。 +func getPreferCatsId(userId, num int) ([]int64, error) { + // STAGE check 一下 key 是否存在。 + // if key != "" { + // redisClient := redis_factory.GetOneRedisClient() + // defer redisClient.ReleaseOneRedisClient() + // if res, err := redisClient.Bool(redisClient.Execute("get", key)); err != nil { + // if res { // 如果 redis 返回的是 1，则代表 prefer 已经耗尽了。 + // return nil, err + // } + // } + // } + // STAGE - 1 模块一，无视条件，获取路遇过的 id 列表；先获取 ID，然后再去查询细节信息。 + encounteredCats, err := model.CreateEncounterFactory("").EncounteredCats(userId, num) + + if err != nil { + // variable.ZapLog.Error("获取用户浏览记录失败", Zap.Error(err)) + return encounteredCats, err + } + return encounteredCats, nil } // v0.1 diff --git a/app/http/validator/web/animal/list.go b/app/http/validator/web/animal/list.go index bd25d9b..936682e 100644 --- a/app/http/validator/web/animal/list.go +++ b/app/http/validator/web/animal/list.go @@ -21,6 +21,7 @@ type List struct { UserId int `form:"user_id" json:"user_id"` Mode string `form:"mode" json:"mode"` // INFO 控制 animal_ctl 的查询模式。 // default: 简单调用 List 函数 || prefer: 优先返回和用户关联度更高的目标。 + Ket string `form:"key" json:"key"` // redis の key 值。 } func (l List) CheckParams(context *gin.Context) { diff --git a/app/model/animal.go b/app/model/animal.go index 79f0ce9..80a1c39 100644 --- a/app/model/animal.go +++ b/app/model/animal.go @@ -51,7 +51,7 @@ func (a *Animal) TableName() string { return "animals" } -func (a *Animal) Show(attrs []string, gender []uint8, breed []uint8, sterilization []uint8, status []uint8, department []uint8, num int, skip int) (temp []Animal) { +func (a *Animal) Show(attrs []string, gender []uint8, breed []uint8, sterilization []uint8, status []uint8, department []uint8, notInIds []int64, num int, skip int) (temp []Animal) { db := a.DB.Table(a.TableName()).Limit(int(num)).Offset(int(skip)).Select(attrs) // 创建条件映射 @@ -65,6 +65,10 @@ func (a *Animal) Show(attrs []string, gender []uint8, breed []uint8, sterilizati db = gorm_v2.BuildWhere(db, conditions) // TIP 这里的 Where 条件连接就很方便了。 + if len(notInIds) > 0 { + db = db.Where("id not in (?)", notInIds) + } + err := db.Find(&temp).Error if err != nil { variable.ZapLog.Error("Animal Show Error", zap.Error(err)) diff --git a/app/model/encounter.go b/app/model/encounter.go index e746a85..c6047cc 100644 --- a/app/model/encounter.go +++ b/app/model/encounter.go @@ -9,7 +9,6 @@ import ( "github.com/gin-gonic/gin" "go.uber.org/zap" "gorm.io/gorm" - ) func CreateEncounterFactory(sqlType string) *Encounter { @@ -99,7 +98,7 @@ func formatEncounterList(rows *gorm.DB) (temp []EncounterList, err error) { } func (e *Encounter) Show(num, skip, user_id int, animals_id []int) (temp []EncounterList) { - // SATGE - 1：build SQL + // STAGE - 1：build SQL var sqlBuilder strings.Builder // 构建基础查询 @@ -122,8 +121,8 @@ WHERE eal.animal_id IN (?)`) // 添加排序和分页 sqlBuilder.WriteString(` - ORDER BY e.updated_at DESC - LIMIT ? OFFSET ? +ORDER BY e.updated_at DESC +LIMIT ? OFFSET ? `) sql := sqlBuilder.String() // 获取到 SQL； @@ -169,3 +168,42 @@ func (e *Encounter) ShowByID(id int64) (temp *Encounter, err error) { // // TODO 4. 然后整合 // return } + +func (e *Encounter) EncounteredCats(user_id, num int) ([]int64, error) { + sql := `SELECT eal.animal_id + FROM encounter_animal_links eal + JOIN encounters e + ON e.id = eal.encounter_id AND e.user_id = ? + ORDER BY e.updated_at DESC + LIMIT ?` + + rows, err := e.Raw(sql, user_id, num).Rows() + if err != nil { + log.Println("查询失败:", err) + return nil, err + } + defer rows.Close() + + // Scan 同时去重。 + var temp []int64 + seen := make(map[int64]bool) + + for rows.Next() { + var animal_id int64 + if err := rows.Scan(&animal_id); err != nil { + log.Println("扫描失败:", err) + return nil, err + } + if !seen[animal_id] { + seen[animal_id] = true + temp = append(temp, animal_id) + } + } + + if err := rows.Err(); err != nil { + log.Println("遍历失败:", err) + return nil, err + } + + return temp, nil +} diff --git a/app/service/animals/curd/animals_curd.go b/app/service/animals/curd/animals_curd.go index 38558d6..3ed210b 100644 --- a/app/service/animals/curd/animals_curd.go +++ b/app/service/animals/curd/animals_curd.go @@ -48,7 +48,7 @@ func getSelectAttrs(attrs string) (validSelectedFields []string) { return } -func (a *AnimalsCurd) List(attrs string, gender string, breed string, sterilization string, status string, department string, num int, skip int, userId int) (temp []model.AnimalWithLikeList) { +func (a *AnimalsCurd) List(attrs string, gender string, breed string, sterilization string, status string, department string, notInIds []int64, num int, skip int, userId int) (temp []model.AnimalWithLikeList) { validSelectedFields := getSelectAttrs(attrs) genderArray := query_handler.StringToUint8Array(gender) breedArray := query_handler.StringToUint8Array(breed) @@ -60,7 +60,7 @@ func (a *AnimalsCurd) List(attrs string, gender string, breed string, sterilizat num = 10 } - animals := model.CreateAnimalFactory("").Show(validSelectedFields, genderArray, breedArray, sterilizationArray, statusArray, departmentArray, num, skip) + animals := model.CreateAnimalFactory("").Show(validSelectedFields, genderArray, breedArray, sterilizationArray, statusArray, departmentArray, notInIds, num, skip) // 状态记录 var likeRes []bool diff --git a/app/service/encounter/curd/encounter_curd.go b/app/service/encounter/curd/encounter_curd.go index 94e5d57..2825ef7 100644 --- a/app/service/encounter/curd/encounter_curd.go +++ b/app/service/encounter/curd/encounter_curd.go @@ -4,6 +4,7 @@ import ( "catface/app/model" "catface/app/utils/query_handler" "strconv" + ) func CreateEncounterCurdFactory() *EncounterCurd { @@ -21,8 +22,8 @@ func (e *EncounterCurd) List(num, skip, user_id int, mode string) (result []mode var likedAnimalIds []int switch mode { - case "liked": - likedAnimalIds = model.CreateAnimalLikeFactory("").LikedCats(user_id) + case "liked": + likedAnimalIds = model.CreateAnimalLikeFactory("").LikedCats(user_id) } result = model.CreateEncounterFactory("").Show(num, skip, user_id, likedAnimalIds) return diff --git a/app/utils/redis_factory/client.go b/app/utils/redis_factory/client.go index 46ab1ae..048c27b 100644 --- a/app/utils/redis_factory/client.go +++ b/app/utils/redis_factory/client.go @@ -10,7 +10,6 @@ import ( "github.com/gomodule/redigo/redis" "go.uber.org/zap" - ) var redisPool *redis.Pool diff --git a/test/redis_test.go b/test/redis_test.go new file mode 100644 index 0000000..4a930fa --- /dev/null +++ b/test/redis_test.go @@ -0,0 +1,135 @@ +package test + +import ( + "catface/app/global/variable" + "catface/app/utils/redis_factory" + _ "catface/bootstrap" + "fmt" + "testing" + "time" + + "go.uber.org/zap" +) + +// 普通的key value +func TestRedisKeyValue(t *testing.T) { + // 从连接池获取一个连接 + redisClient := redis_factory.GetOneRedisClient() + + // set 命令, 因为 set key value 在redis客户端执行以后返回的是 ok，所以取回结果就应该是 string 格式 + res, err := redisClient.String(redisClient.Execute("set", "key2020", "value202022")) + if err != nil { + t.Errorf("单元测试失败,%s\n", err.Error()) + } else { + variable.ZapLog.Info("Info 日志", zap.String("key2020", res)) + } + // get 命令，分为两步：1.执行get命令 2.将结果转为需要的格式 + if res, err = redisClient.String(redisClient.Execute("get", "key2020")); err != nil { + t.Errorf("单元测试失败,%s\n", err.Error()) + } + variable.ZapLog.Info("get key2020 ", zap.String("key2020", res)) + //操作完毕记得释放连接，官方明确说，redis使用完毕，必须释放 + redisClient.ReleaseOneRedisClient() +} + +func TestRedisKeyInt64(t *testing.T) { + redisClient := redis_factory.GetOneRedisClient() + defer redisClient.ReleaseOneRedisClient() + + id := variable.SnowFlake.GetId() + + res, err := redisClient.String(redisClient.Execute("set", id, 1)) + if err != nil { + t.Errorf("单元测试失败,%s\n", err.Error()) + } + + if res, err = redisClient.String(redisClient.Execute("get", id)); err != nil { + t.Errorf("单元测试失败,%s\n", err.Error()) + } else { + t.Logf("单元测试通过,%s\n", res) + } +} + +// hash 键、值 +func TestRedisHashKey(t *testing.T) { + + redisClient := redis_factory.GetOneRedisClient() + + // hash键 set 命令, 因为 hSet h_key key value 在redis客户端执行以后返回的是 1 或者 0，所以按照int64格式取回 + res, err := redisClient.Int64(redisClient.Execute("hSet", "h_key2020", "hKey2020", "value2020_hash")) + if err != nil { + t.Errorf("单元测试失败,%s\n", err.Error()) + } else { + fmt.Println(res) + } + // hash键 get 命令，分为两步：1.执行get命令 2.将结果转为需要的格式 + res2, err := redisClient.String(redisClient.Execute("hGet", "h_key2020", "hKey2020")) + if err != nil { + t.Errorf("单元测试失败,%s\n", err.Error()) + } + fmt.Println(res2) + //官方明确说，redis使用完毕，必须释放 + redisClient.ReleaseOneRedisClient() +} + +// 测试 redis 连接池 +func TestRedisConnPool(t *testing.T) { + + for i := 1; i <= 20; i++ { + go func() { + redisClient := redis_factory.GetOneRedisClient() + fmt.Printf("获取的redis数据库连接池地址：%p\n", redisClient) + time.Sleep(time.Second * 10) + fmt.Printf("阻塞过程中，您可以通过redis命令 client list 查看链接的客户端") + redisClient.ReleaseOneRedisClient() // 释放从连接池获取的连接 + }() + } + time.Sleep(time.Second * 20) +} + +// 测试redis 网络中断自动重连机制 +func TestRedisReConn(t *testing.T) { + redisClient := redis_factory.GetOneRedisClient() + res, err := redisClient.String(redisClient.Execute("set", "key2020", "测试网络抖动，自动重连机制")) + if err != nil { + t.Errorf("单元测试失败,%s\n", err.Error()) + } else { + variable.ZapLog.Info("Info 日志", zap.String("key2020", res)) + } + //官方明确说，redis使用完毕，必须释放 + redisClient.ReleaseOneRedisClient() + + // 以上内容输出后 ， 拔掉网线, 模拟短暂的网络抖动 + t.Log("请在 10秒之内拔掉网线") + time.Sleep(time.Second * 10) + // 断网情况下就会自动进行重连 + redisClient = redis_factory.GetOneRedisClient() + if res, err = redisClient.String(redisClient.Execute("get", "key2020")); err != nil { + t.Errorf("单元测试失败,%s\n", err.Error()) + } else { + t.Log("获取的值：", res) + } + redisClient.ReleaseOneRedisClient() +} + +// 测试返回值为多值的情况 +func TestRedisMulti(t *testing.T) { + redisClient := redis_factory.GetOneRedisClient() + + if _, err := redisClient.String(redisClient.Execute("multi")); err == nil { + redisClient.Execute("hset", "mobile", "xiaomi", "1999") + redisClient.Execute("hset", "mobile", "oppo", "2999") + redisClient.Execute("hset", "mobile", "iphone", "3999") + + if strs, err := redisClient.Int64s(redisClient.Execute("exec")); err == nil { + t.Logf("直接输出切片：%#+v\n", strs) + } else { + t.Errorf(err.Error()) + } + } else { + t.Errorf(err.Error()) + } + redisClient.ReleaseOneRedisClient() +} + +// 其他请参照以上示例即可 diff --git a/test/snowflake_test.go b/test/snowflake_test.go new file mode 100644 index 0000000..fdb8555 --- /dev/null +++ b/test/snowflake_test.go @@ -0,0 +1,58 @@ +package test + +import ( + "catface/app/global/variable" + _ "catface/bootstrap" + "fmt" + "sync" + "testing" +) + +// 雪花算法单元测试 +func TestSnowFlake(t *testing.T) { + num := 3 + // 并发 3万 测试，实际业务场景中，并发是不可能达到 3万 这个值的 + var slice1 []int64 + var vMuext sync.Mutex + var wg sync.WaitGroup + wg.Add(num) + + for i := 1; i <= num; i++ { + go func() { + defer wg.Done() + //加锁操作主要是为了保证切片（[]int64）的并发安全， + //我们本次测试的核心目的是雪花算法生成的ID必须是唯一的 + vMuext.Lock() + slice1 = append(slice1, variable.SnowFlake.GetId()) + fmt.Println(slice1) + vMuext.Unlock() + //fmt.Printf("%d\n", variable.SnowFlake.GetId()) + }() + } + + wg.Wait() + + if lastLen := len(RemoveRepeatedElement(slice1)); lastLen == num { + t.Log("单元测试OK") + } else { + t.Errorf("雪花算法单元测试失败,并发 3万 生成的id经过去重之后，小于预期个数，去重后的个数：%d\n", lastLen) + } +} + +// 切片去重 +func RemoveRepeatedElement(arr []int64) (newArr []int64) { + newArr = make([]int64, 0) + for i := 0; i < len(arr); i++ { + repeat := false + for j := i + 1; j < len(arr); j++ { + if arr[i] == arr[j] { + repeat = true + break + } + } + if !repeat { + newArr = append(newArr, arr[i]) + } + } + return +}