From 5412c07c57142ae2a6d1ea46593a63d09966ee2c Mon Sep 17 00:00:00 2001 From: Sven Schmalle Date: Tue, 17 Dec 2019 16:52:54 +0100 Subject: [PATCH] =?UTF-8?q?Volltextsuche=20via=20blevesearch=20hinzugef?= =?UTF-8?q?=C3=BCgt?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 7 ++-- config.json | 3 +- main.go | 86 +++++++++++++++++++++++++++++++++++++++++++++++--- web/index.html | 34 +++++++++++++++++++- 4 files changed, 122 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 8d243be..1b12134 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,8 @@ Eine `config.json` könnte z.B. wie folgt aussehen: { "Host":"http://127.0.0.1", "Port":"8000", - "DataPath":"./data" + "DataPath":"./data", + "DataPathFTS":"./FTSData" } ``` @@ -24,7 +25,9 @@ Eine `config.json` könnte z.B. wie folgt aussehen: Zum kompilieren folgende Go-Abhängigkeiten installieren: - `go get github.com/gorilla/mux` - `go get gopkg.in/src-d/go-git.v4` - - 'go get github.com/mandolyte/mdtopdf' + - `go get github.com/mandolyte/mdtopdf` + - `go get github.com/blevesearch/bleve` + Die Build-Scripte liegen im Unterordner `build` diff --git a/config.json b/config.json index 9d07f3f..cf5b96e 100644 --- a/config.json +++ b/config.json @@ -1,5 +1,6 @@ { "Host":"http://127.0.0.1", "Port":"8000", - "DataPath":"./data" + "DataPath":"./data", + "DataPathFTS":"./FTSData" } \ No newline at end of file diff --git a/main.go b/main.go index 0198037..e563d6b 100644 --- a/main.go +++ b/main.go @@ -14,6 +14,7 @@ import ( "strings" "time" + "github.com/blevesearch/bleve" "github.com/gorilla/mux" "github.com/mandolyte/mdtopdf" "gopkg.in/src-d/go-git.v4" @@ -23,9 +24,10 @@ import ( var config = readConfig("") type Configuration struct { - Host string - Port string - DataPath string + Host string + Port string + DataPath string + DataPathFTS string } // our main function @@ -35,6 +37,7 @@ func main() { fmt.Println("Host: " + config.Host) fmt.Println("Post: " + config.Port) fmt.Println("DataPath: " + config.DataPath) + fmt.Println("DataPathFTS: " + config.DataPathFTS) // DataPath-Verzeichnis anlegen, wenn es noch nicht existiert if !directoryExists(config.DataPath) { @@ -64,9 +67,42 @@ func main() { GitCommit("Initial Wiki Commit") } + // Volltextsuche + if config.DataPathFTS != "" { + if strings.HasPrefix(config.DataPathFTS, config.DataPath) { + fmt.Println("FTS disabled (Please don´t put the DataPathFTS inside the DataPath)") + } else { + fmt.Println("FTS enabled") + if !directoryExists(config.DataPathFTS) { + fmt.Println("Create new FTS Data in " + config.DataPathFTS) + mapping := bleve.NewIndexMapping() + + index, err := bleve.New(config.DataPathFTS, mapping) + check(err) + + filepath.Walk(config.DataPath, func(filepath string, info os.FileInfo, err error) error { + tmpPath := strings.Replace(filepath, "\\", "/", -1) + tmpPath = strings.Replace(tmpPath, "data/", "", -1) + + if !info.IsDir() && !strings.HasPrefix(tmpPath, ".git") { + b, err := ioutil.ReadFile(path.Join(config.DataPath, tmpPath)) + check(err) + err = index.Index(tmpPath, string(b)) + check(err) + fmt.Println("Indexed: " + info.Name()) + } + return nil + }) + + index.Close() + } + } + } + router := mux.NewRouter() router.HandleFunc("/_api/md/{pagename:.*}", getRawPage).Methods("GET") router.HandleFunc("/_api/pdf/{pagename:.*}", getPDFPage).Methods("GET") + router.HandleFunc("/_api/fts/{searchterm:.*}", getFTS).Methods("GET") router.HandleFunc("/{pagename:.*}", getHTMLPage).Methods("GET") router.HandleFunc("/{pagename:.*}", postHTMLPage).Methods("POST") @@ -202,11 +238,37 @@ func postHTMLPage(w http.ResponseWriter, r *http.Request) { // Git Commit GitCommit("Auto-Commit Page: " + pPageName) + // FTS Index + BleveIndex(pPageName) json.NewEncoder(w).Encode("OK") } +func getFTS(w http.ResponseWriter, r *http.Request) { + + params := mux.Vars(r) + pSearchterm := params["searchterm"] + + if config.DataPathFTS != "" { + if directoryExists(config.DataPathFTS) { + + index, err := bleve.Open(config.DataPathFTS) + check(err) + + query := bleve.NewQueryStringQuery(pSearchterm) + search := bleve.NewSearchRequest(query) + search.Highlight = bleve.NewHighlight() + search.Size = 30 + searchResults, err := index.Search(search) + index.Close() + + //fmt.Println("SR: " + searchResults.String()) + json.NewEncoder(w).Encode(searchResults.Hits) + } + } +} + //-------------------------------------------------------------------------- // Typen //-------------------------------------------------------------------------- @@ -249,7 +311,7 @@ func directoryExists(filename string) bool { func readConfig(filename string) *Configuration { // initialize conf with default values. - conf := &Configuration{Host: "http://127.0.0.1", Port: "8000", DataPath: "./data"} + conf := &Configuration{Host: "http://127.0.0.1", Port: "8000", DataPath: "./data", DataPathFTS: ""} b, err := ioutil.ReadFile("./config.json") if err != nil { @@ -295,3 +357,19 @@ func GitCommit(CommitText string) { //check(err) //fmt.Println(obj) } + +func BleveIndex(PageName string) { + if config.DataPathFTS != "" { + if directoryExists(config.DataPathFTS) { + + index, err := bleve.Open(config.DataPathFTS) + check(err) + + b, err := ioutil.ReadFile(path.Join(config.DataPath, PageName)) + check(err) + err = index.Index(PageName, string(b)) + check(err) + index.Close() + } + } +} diff --git a/web/index.html b/web/index.html index 7a5b539..ef996bd 100644 --- a/web/index.html +++ b/web/index.html @@ -120,6 +120,11 @@
  • Save
  • PDF
    • + @@ -224,7 +229,34 @@ } }); - + $('#srch-term').keypress(function (e) { + if (e.which == 13) { + //alert($('#srch-term').val()); + + $.ajax({ + method: "GET", + contentType:'application/json; charset=utf-8', + url: '/_api/fts/'+$('#srch-term').val(), + dataType: "json", + data: "", + success: function(content){ + + html = "

    Suche

    Der Suchbegriff \""+$('#srch-term').val()+"\" wurde "+content.length +" mal in folgenden Seiten gefunden:
    " + //html = MD2HTMLConverter.makeHtml(content+$("#outputsidebar").html()); + //console.log(content); + $("#outputdiv").html(html); + $("#outputdiv").show(); + + content.forEach(function (item, index) { + console.log(item.id); + var page = item.id.replace(/.md/, ''); + $("#outputdiv").append("/"+page+"
    "); + }); + } + }); + return false; + } + }); }); function EditPage() {