分布式搜索引擎，你真的懂吗?

type Index struct {    Fields []string    DocMap map[string][]int //词到文档列表映射}
func IndexDoc(doc *Document) {        index := &Index{}        //分词     tokens := Tokenize(doc.Content)          //构建倒排索引    for _, token := range tokens {        index.DocMap[token] = append(index.DocMap[token], doc.ID)      }}
func SearchQuery(query) {        //解析查询词    tokens := Tokenize(query)           //交集算法获取共现文档    //返回文档id列表    return IntersecForTokens(index.DocMap, tokens)}

//基于词典的中文分词import "github.com/go-ego/gse"
func Tokenize(text string) []string  {
    segmenter := gse.New("dict.txt")        // 分词    tokens := segmenter.Segment([]byte(text))        return tokens}

//一致性hash分配import "github.com/cch123/elastics/hashring"
var nodes = []string{"server1","server2"...}ring := hashring.New(nodes)
func DistributeDoc(doc *Document) {        //根据id选择节点    node = ring.GetNode(doc.id)          //索引文档    indexDoc(doc, node)  }

import "github.com/patrickmn/go-cache"
type QueryCache struct {  cache *cache.Cache //词到结果缓存}
//获取查询结果  func (c *QueryCache)Search(query) {        if c.cache.Has(query) {        return cache.Get(query)     }        //缓存未命中,搜索索引    result = indexSearch(query)         c.cache.Set(query, result)    return result}

func QueryRewrite(query string) string {
    rewriteRules :=map[string]string {          "宝马" : "汽车",        "招商银行" : "银行"    }
    if rewrite, ok := rules[query]; ok {        return rewrite    }
    return query}

type RankedResult struct {    DocId int      Score float32 //相关度  }
func Rank(results []int, tokens []string) {        scoredList := make([]RankedResult,0) 
    for _,docId := range results {         score = 0                if frequency[docId][token] > 0 {            score ++        }                 //词出现在标题权重更高         if postion[docId][token] == "title" {            score += 5        }                //文档自身权重分        score += docWeight[docId]                scored := RankedResult(docId, score)        scoredList = append(scoredList, scored)    } 
    //按相关度降序排序并返回     sort.Sort(scoredList)    return scoredList}

// bleve 建立索引示例
import (        "github.com/blevesearch/bleve"        "github.com/blevesearch/bleve/index/upsidedown")
// 创建倒排索引实例mapping := bleve.NewIndexMapping() index, _ := bleve.New("example.bleve",mapping)  
//索引文档data := []byte("Bleve builds indexes for text search.")index.Index(1,data)
// Term查询query := bleve.NewTermQuery("Bleve")request := bleve.NewSearchRequest(query)result,_:= index.Search(request)

// riot 分布索引文档
import (      "github.com/go-ego/riot"    "github.com/go-ego/riot/types")
var crawler = riot.New("zh")crawler.Index(types.DocData{    Id:   1,     Text: "文档内容",})
//搜索查询import (    "github.com/go-ego/riot"    "github.com/go-ego/riot/types")
var searcher = riot.New("zh")searcher.Search(types.SearchRequest{    Text: "关键词",     RankOpts: &types.RankOpts{          OutputOffset: 5,    },})

// 连接 ElasticSearchimport (    "github.com/olivere/elastic")
client, err := elastic.NewClient(elastic.SetURL("http://ip:port"))
// 索引文档client.Index().    Index("twitter").    Type("tweet").    Id("1").    BodyJson(tweet).    Do(context.Background())
//搜索searchResult, err := client.Search().    Index("twitter").      Query(elastic.NewTermQuery("user", "kimchy")).      From(0).Size(10).    Pretty(true).    Do(context.Background())