diff --git a/app/http/controller/web/search_controller.go b/app/http/controller/web/search_controller.go index 64795de..35d7611 100644 --- a/app/http/controller/web/search_controller.go +++ b/app/http/controller/web/search_controller.go @@ -2,8 +2,9 @@ package web import ( "catface/app/global/consts" - "catface/app/model" "catface/app/model_es" + animal_curd "catface/app/service/animals/curd" + encouner_curd "catface/app/service/encounter/curd" "catface/app/utils/response" "github.com/gin-gonic/gin" @@ -20,18 +21,12 @@ type Search struct { func (s *Search) SearchAll(context *gin.Context) { query := context.GetString(consts.ValidatorPrefix + "query") - var animals []model.Animal - var encounters []model.Encounter - // 1. Animal Name // TODO 增加字段的过滤,看前端了。 - animals = model.CreateAnimalFactory("").ShowByName(query) + // animals = model.CreateAnimalFactory("").ShowByName(query) + animals := animal_curd.CreateAnimalsCurdFactory().MatchAll(query, 3) // 2. Encounter - _, _ = model_es.CreateEncounterESFactory(nil).QueryDocumentsMatchAll(query) - - // if len(encounterIds) > 0 { - // encounters = model.CreateEncounterFactory("").ShowByIDs(encounterIds) - // } + encounters := encouner_curd.CreateEncounterCurdFactory().MatchAll(query, 3) // 3. Knowledge knowledges, _ := model_es.CreateKnowledgeESFactory().QueryDocumentsMatchAll(query, 3) diff --git a/app/model/animal.go b/app/model/animal.go index 0e0860b..db74c2b 100644 --- a/app/model/animal.go +++ b/app/model/animal.go @@ -15,19 +15,20 @@ func CreateAnimalFactory(sqlType string) *Animal { type Animal struct { // UPDATE 或者这里都应该采取外键连接? - BaseModel // 假设 BaseModel 中不需要添加 omitempty 标签 - Name string `gorm:"type:varchar(20)" json:"name,omitempty"` // 名称 - Birthday string `gorm:"size:10" json:"birthday,omitempty"` // 生日;就简单存string就好 - Gender uint8 `gorm:"default:1" json:"gender,omitempty"` // 性别 - Breed uint8 `gorm:"default:1" json:"breed,omitempty"` // 品种 - Sterilization uint8 `gorm:"default:1" json:"sterilization,omitempty"` // 1 不明 2 未绝育 3 已绝育 - Vaccination uint8 `gorm:"default:1" json:"vaccination,omitempty"` // 免疫状态 - Deworming uint8 `gorm:"default:1" json:"deworming,omitempty"` // 驱虫状态 - NickNames string `gorm:"type:varchar(31)" json:"nick_names,omitempty"` // 别称,辅助查询;存储上采取 , 间隔符的方式; VARCHAR 会比较合适 - NickNamesList []string `gorm:"-" json:"nick_names_list,omitempty"` - Status uint8 `gorm:"default:1" json:"status,omitempty"` // 状态 - Description string `gorm:"column:description;type:varchar(255)" json:"description,omitempty"` // 简明介绍 - Tags string `json:"tags,omitempty"` + BaseModel // 假设 BaseModel 中不需要添加 omitempty 标签 + Name string `gorm:"type:varchar(20)" json:"name,omitempty"` // 名称 + Birthday string `gorm:"size:10" json:"birthday,omitempty"` // 生日;就简单存string就好 + Gender uint8 `gorm:"default:1" json:"gender,omitempty"` // 性别 + Breed uint8 `gorm:"default:1" json:"breed,omitempty"` // 品种 + Sterilization uint8 `gorm:"default:1" json:"sterilization,omitempty"` // 1 不明 2 未绝育 3 已绝育 + Vaccination uint8 `gorm:"default:1" json:"vaccination,omitempty"` // 免疫状态 + Deworming uint8 `gorm:"default:1" json:"deworming,omitempty"` // 驱虫状态 + NickNames string `gorm:"type:varchar(31)" json:"nick_names,omitempty"` // 别称,辅助查询;存储上采取 , 间隔符的方式; VARCHAR 会比较合适 + NickNamesList []string `gorm:"-" json:"nick_names_list,omitempty"` + NickNamesHighlight []string `gorm:"-" json:"nick_names_highlight,omitempty"` // INFO 配合 ES + Status uint8 `gorm:"default:1" json:"status,omitempty"` // 状态 + Description string `gorm:"column:description;type:varchar(255)" json:"description,omitempty"` // 简明介绍 + Tags string `json:"tags,omitempty"` // TAG imaegs Avatar string `gorm:"type:varchar(50)" json:"avatar,omitempty"` // 缩略图 url,为 Go 获取 Photo 之后压缩处理后的图像,单独存储。 AvatarHeight uint16 `json:"avatar_height,omitempty"` // 为了方便前端在加载图像前的骨架图 & 瀑布流展示。 // INFO 暂时没用到 diff --git a/app/model/encounter.go b/app/model/encounter.go index b1e5ce4..4371ad4 100644 --- a/app/model/encounter.go +++ b/app/model/encounter.go @@ -22,11 +22,12 @@ type Encounter struct { // Encounter 或者称为 post,指的就是 Human 单 UsersModel *UsersModel `json:"users_model,omitempty"` // INFO 由于 Detail 返回空子段有些麻烦,先尝试采用指针。 // AnimalsId string `gorm:"size:20" json:"animals_id"` // 关联对象存在上限 // INFO 还是采取分表,方便查询。 - Title string `gorm:"size:20;column:title" json:"title"` - Content string `json:"content"` - Level uint8 `json:"level" gorm:"column:level;default:1"` - Tags string `json:"tags,omitempty" gorm:"column:tags;size:50"` - TagsList []string `gorm:"-" json:"tags_list,omitempty"` + Title string `gorm:"size:20;column:title" json:"title"` + Content string `json:"content"` + Level uint8 `json:"level" gorm:"column:level;default:1"` + Tags string `json:"tags,omitempty" gorm:"column:tags;size:50"` + TagsList []string `gorm:"-" json:"tags_list,omitempty"` + TagsHighlight []string `gorm:"-" json:"tags_highlight,omitempty"` // TAG Avatar 最好是压缩后的备份图像 Avatar string `gorm:"type:varchar(50)" json:"avatar,omitempty"` // 缩略图 url,为 Go 获取 Photo 之后压缩处理后的图像,单独存储。 diff --git a/app/model/users.go b/app/model/users.go index 16a7c12..d0f88e3 100644 --- a/app/model/users.go +++ b/app/model/users.go @@ -362,3 +362,14 @@ func (u *UsersModel) ShowByID(id int64, attrs ...string) (temp *UsersModel, err } return } + +func (u *UsersModel) ShowByIDs(ids []int64, attrs ...string) (temp []UsersModel) { + db := u.DB.Table(u.TableName()) + + if len(attrs) > 0 { + db = db.Select(attrs) + } + + db.Where("id in (?)", ids).Find(&temp) + return +} diff --git a/app/model_es/animal.go b/app/model_es/animal.go index 14b1dbf..f5e1e7f 100644 --- a/app/model_es/animal.go +++ b/app/model_es/animal.go @@ -4,6 +4,8 @@ import ( "bytes" "catface/app/global/variable" "catface/app/model" + "catface/app/utils/data_bind" + "catface/app/utils/model_handler" "context" "encoding/json" "fmt" @@ -28,6 +30,9 @@ type Animal struct { Name string `json:"name"` NickNames []string `json:"nick_names"` Description string `json:"description"` + + // After handler + NickNamesHighlight []string `json:"nick_names_highlight"` } func (a *Animal) IndexName() string { @@ -70,3 +75,51 @@ func (a *Animal) InsertDocument() error { return nil } + +func (a *Animal) QueryDocumentsMatchAll(query string, num int) ([]Animal, error) { + body := fmt.Sprintf(`{ + "size": %d, + "query": { + "bool": { + "should": [ + { "match": {"name": "%s" }}, + { "match": {"nick_names": "%s" }}, + { "match": {"description": "%s" }} + ] + } + }, + "highlight": { + "pre_tags": [""], + "post_tags": [""], + "fields": { + "name": {}, + "nick_names": { + "pre_tags": [""], + "post_tags": [""] + }, + "description": { + "fragment_size" : 15 + } + } + } +}`, num, query, query, query) + + hits, err := model_handler.SearchRequest(body, a.IndexName()) + if err != nil { + return nil, err + } + + var animals []Animal + for _, hit := range hits { + data := model_handler.MergeSouceWithHighlight(hit.(map[string]interface{})) + + var animal Animal + if err := data_bind.ShouldBindFormMapToModel(data, &animal); err != nil { + continue + } + + animals = append(animals, animal) + } + + return animals, nil +} diff --git a/app/model_es/encounter.go b/app/model_es/encounter.go index 2a25152..895e639 100644 --- a/app/model_es/encounter.go +++ b/app/model_es/encounter.go @@ -4,10 +4,11 @@ import ( "bytes" "catface/app/global/variable" "catface/app/model" + "catface/app/utils/data_bind" + "catface/app/utils/model_handler" "context" "encoding/json" "fmt" - "strings" "github.com/elastic/go-elasticsearch/v8" "github.com/elastic/go-elasticsearch/v8/esapi" @@ -33,6 +34,8 @@ type Encounter struct { Title string `json:"title"` Content string `json:"content"` Tags []string `json:"tags"` + + TagsHighlight []string `json:"tags_highlight"` } func (e *Encounter) IndexName() string { @@ -128,92 +131,50 @@ func (e *Encounter) UpdateDocument(client *elasticsearch.Client, encounter *Enco * @param {string} query * @return {*} 对应 Encounter 的 id,然后交给 MySQL 来查询详细的信息? */ -func (e *Encounter) QueryDocumentsMatchAll(query string) ([]Encounter, error) { - ctx := context.Background() +func (e *Encounter) QueryDocumentsMatchAll(query string, num int) ([]Encounter, error) { + body := fmt.Sprintf(`{ + "size": %d, + "query": { + "bool": { + "should": [ + {"match": {"tags": "%s"}}, + {"match": {"content": "%s"}}, + {"match": {"title": "%s"}} + ] + } + }, + "highlight": { + "pre_tags": [""], + "post_tags": [""], + "fields": { + "title": {}, + "content": { + "fragment_size" : 15 + }, + "tags": { + "pre_tags": [""], + "post_tags": [""] + } + } + } +}`, num, query, query, query) - // 创建查询请求 - req := esapi.SearchRequest{ // UPDATE 同时实现查询高亮? - Index: []string{e.IndexName()}, - // INFO 采取高光的设定,所以还是用 ES 的返回值会比较好; "_source": ["id"], - Body: strings.NewReader(fmt.Sprintf(`{ - "query": { - "bool": { - "should": [ - { - "match": { "title": "%s" } - }, - { - "match": { "content": "%s" } - } - ] - } - } - }`, query, query)), - } - - // 发送请求 - res, err := req.Do(ctx, variable.ElasticClient) + hits, err := model_handler.SearchRequest(body, e.IndexName()) if err != nil { return nil, err } - defer res.Body.Close() - if res.IsError() { - var e map[string]interface{} - if err := json.NewDecoder(res.Body).Decode(&e); err != nil { - return nil, fmt.Errorf("error parsing the response body: %s", err) - } else { - return nil, fmt.Errorf("[%s] %s: %s", - res.Status(), - e["error"].(map[string]interface{})["type"], - e["error"].(map[string]interface{})["reason"], - ) - } - } - - // 解析响应 - var r map[string]interface{} - if err := json.NewDecoder(res.Body).Decode(&r); err != nil { - return nil, err - } - - // 提取命中结果 - hits, ok := r["hits"].(map[string]interface{})["hits"].([]interface{}) - if !ok { - return nil, fmt.Errorf("error extracting hits from response") - } - - // // 转换为 id 切片 - // var ids []int64 - // for _, hit := range hits { - // hitMap := hit.(map[string]interface{})["_source"].(map[string]interface{}) - // id := int64(hitMap["id"].(float64)) - // ids = append(ids, id) - // } - // return ids, nil - - // 转换为 Encounter 切片 - var encounters []*Encounter + var encounters []Encounter for _, hit := range hits { - hitMap := hit.(map[string]interface{}) - source := hitMap["_source"].(map[string]interface{}) - // highlight := hitMap["highlight"].(map[string]interface{}) + data := model_handler.MergeSouceWithHighlight(hit.(map[string]interface{})) - // TIP 将 []interface{} 转换为 []string - tagsInterface := source["tags"].([]interface{}) - tags := make([]string, len(tagsInterface)) - for i, tag := range tagsInterface { - tags[i] = tag.(string) + var encounter Encounter + if err := data_bind.ShouldBindFormMapToModel(data, &encounter); err != nil { + continue } - encounter := &Encounter{ - Id: int64(source["id"].(float64)), - Title: source["title"].(string), - Content: source["content"].(string), - Tags: tags, - } encounters = append(encounters, encounter) } - return []Encounter{}, nil + return encounters, nil } diff --git a/app/model_es/knowledge.go b/app/model_es/knowledge.go index 71d8594..24700a9 100644 --- a/app/model_es/knowledge.go +++ b/app/model_es/knowledge.go @@ -218,6 +218,7 @@ func (k *Knowledge) QueryDocumentsMatchAll(query string, num int) ([]Knowledge, highlight := hitMap["highlight"].(map[string]interface{}) for k, v := range highlight { + // INFO Knowledge 暂时不涉及 keywords 类型,就先这样处理。 source[k] = model_handler.TransStringSliceToString(v.([]interface{})) } diff --git a/app/service/animals/curd/animals_curd.go b/app/service/animals/curd/animals_curd.go index 571ccbd..78c0586 100644 --- a/app/service/animals/curd/animals_curd.go +++ b/app/service/animals/curd/animals_curd.go @@ -2,6 +2,7 @@ package curd import ( "catface/app/model" + "catface/app/model_es" "catface/app/utils/gorm_v2" "catface/app/utils/model_handler" "catface/app/utils/query_handler" @@ -122,3 +123,35 @@ func (a *AnimalsCurd) Detail(id string) *model.Animal { return model.CreateAnimalFactory("mysql").ShowByID(int64(idInt)) } + +func (a *AnimalsCurd) MatchAll(query string, num int) (tmp []model.Animal) { + // STAGE 1. ES 查询 + animalsFromES, err := model_es.CreateAnimalESFactory(nil).QueryDocumentsMatchAll(query, num) + if err != nil { + fmt.Println("ES Query error:", err) + return nil + } + + var ids []int64 + for _, animal := range animalsFromES { + ids = append(ids, animal.Id) + } + + // STAGE 2. MySQL 补充信息 + animalsFromSQL := model.CreateAnimalFactory("").ShowByIDs(ids, "id", "avatar") + + // 3. 合并信息 + for _, animalFromES := range animalsFromES { + for _, animal := range animalsFromSQL { + if animal.Id == animalFromES.Id { + animal.NickNamesList = animalFromES.NickNames + animal.NickNamesHighlight = animalFromES.NickNamesHighlight + animal.Description = animalFromES.Description + animal.Name = animalFromES.Name + tmp = append(tmp, animal) + } + } + } + + return +} diff --git a/app/service/encounter/curd/encounter_curd.go b/app/service/encounter/curd/encounter_curd.go index e09ad0f..9ac50bb 100644 --- a/app/service/encounter/curd/encounter_curd.go +++ b/app/service/encounter/curd/encounter_curd.go @@ -2,6 +2,7 @@ package curd import ( "catface/app/model" + "catface/app/model_es" "catface/app/utils/query_handler" "strconv" ) @@ -64,3 +65,45 @@ func (e *EncounterCurd) Detail(id string) *model.EncounterDetail { Animals: animals, } } + +func (e *EncounterCurd) MatchAll(query string, num int) (tmp []model.Encounter) { + // 1. encounter ES + encountersFromES, err := model_es.CreateEncounterESFactory(nil).QueryDocumentsMatchAll(query, num) + if err != nil || len(encountersFromES) == 0 { + return nil + } + + var ids []int64 + for _, encounter := range encountersFromES { + ids = append(ids, encounter.Id) + } + + // 2. encounter SQL + encountersFromSQL := model.CreateEncounterFactory("").ShowByIDs(ids, "id", "avatar", "user_id") + + // 3. users + ids = nil + for _, encounter := range encountersFromSQL { + ids = append(ids, encounter.UsersModelId) + } + users := model.CreateUserFactory("").ShowByIDs(ids, "user_avatar", "user_name", "id") + + // end. Merge + for _, enencountersFromES := range encountersFromES { + for _, encounter := range encountersFromSQL { + for _, user := range users { + if encounter.Id == enencountersFromES.Id && encounter.UsersModelId == user.Id { + encounter.TagsList = enencountersFromES.Tags + encounter.TagsHighlight = enencountersFromES.TagsHighlight + encounter.Title = enencountersFromES.Title + encounter.Content = enencountersFromES.Content + + encounter.UsersModel = &user + tmp = append(tmp, encounter) + } + } + } + } + + return +} diff --git a/app/utils/data_bind/formdata_to_model.go b/app/utils/data_bind/formdata_to_model.go index 640d434..42f3e3e 100644 --- a/app/utils/data_bind/formdata_to_model.go +++ b/app/utils/data_bind/formdata_to_model.go @@ -109,7 +109,7 @@ func ShouldBindFormMapToModel(m map[string]interface{}, modelStruct interface{}) func fieldSetValueByMap(m map[string]interface{}, valueOf reflect.Value, typeOf reflect.Type, colIndex int) { relaKey := typeOf.Field(colIndex).Tag.Get("json") - if relaKey != "-" { + if relaKey != "-" && m[relaKey] != nil { switch typeOf.Field(colIndex).Type.Kind() { case reflect.Int, reflect.Int16, reflect.Int32, reflect.Int64: valueOf.Field(colIndex).SetInt(int64(m[relaKey].(float64))) diff --git a/app/utils/model_handler/model_es_handler.go b/app/utils/model_handler/model_es_handler.go index a5e3659..64e13dd 100644 --- a/app/utils/model_handler/model_es_handler.go +++ b/app/utils/model_handler/model_es_handler.go @@ -1,9 +1,19 @@ package model_handler +import ( + "catface/app/global/variable" + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/elastic/go-elasticsearch/v8/esapi" +) + /** * @description: 用于处理 ES-highlight 模块分析出来的 []String. * @param {[]interface{}} strs - * @return {*} + * @return {*} 将 []String 还原为 String,这里就是简单的拼接了一下。 */ func TransStringSliceToString(strs []interface{}) string { var result string @@ -14,3 +24,71 @@ func TransStringSliceToString(strs []interface{}) string { } return result } + +// func concatKeywordsToSlice(highlightKeyword string, oriKeywords []string) []string { +// return append(oriKeywords, highlightKeyword) +// } + +func MergeSouceWithHighlight(hit map[string]interface{}) map[string]interface{} { + // 1. Get data + source := hit["_source"].(map[string]interface{}) + highlight := hit["highlight"].(map[string]interface{}) + + // 2. Merge data + for k, v := range highlight { + if _, ok := source[k]; ok { + switch source[k].(type) { + case string: + source[k] = TransStringSliceToString(v.([]interface{})) + case []interface{}: + source[k+"_highlight"] = v // TODO 过滤,交给前端? + } + } + } + return source +} + +/** + * @description: 对 ES 发送的请求,返回结果。 + * @param {string} body + * @param {string} index + * @return {*} + */ +func SearchRequest(body string, index string) ([]interface{}, error) { + ctx := context.Background() + + req := esapi.SearchRequest{ + Index: []string{index}, + Body: strings.NewReader(body), + } + + res, err := req.Do(ctx, variable.ElasticClient) + if err != nil { + return nil, err + } + defer res.Body.Close() + + if res.IsError() { + var k map[string]interface{} + if err := json.NewDecoder(res.Body).Decode(&k); err != nil { + return nil, fmt.Errorf("error parsing the response body: %s", err) + } else { + return nil, fmt.Errorf("[%s] %s: %s", + res.Status(), + k["error"].(map[string]interface{})["type"], + k["error"].(map[string]interface{})["reason"], + ) + } + } + + var result map[string]interface{} + if err := json.NewDecoder(res.Body).Decode(&result); err != nil { + return nil, err + } + + hits, ok := result["hits"].(map[string]interface{})["hits"].([]interface{}) + if !ok { + return nil, fmt.Errorf("error extracting hits from response") + } + return hits, nil +}