about summary refs log tree commit diff stats
path: root/internal/search/search.go
diff options
context:
space:
mode:
Diffstat (limited to 'internal/search/search.go')
-rw-r--r--internal/search/search.go150
1 files changed, 36 insertions, 114 deletions
diff --git a/internal/search/search.go b/internal/search/search.go
index 97d8404..92afdfb 100644
--- a/internal/search/search.go
+++ b/internal/search/search.go
@@ -4,151 +4,73 @@ import (
 	"bytes"
 	"context"
 	"encoding/gob"
-	"log"
-	"os"
 	"path"
 	"searchix/internal/options"
 
-	"github.com/bcicen/jstream"
 	"github.com/blevesearch/bleve/v2"
-	"github.com/blevesearch/bleve/v2/analysis/analyzer/custom"
-	"github.com/blevesearch/bleve/v2/analysis/token/camelcase"
-	"github.com/blevesearch/bleve/v2/analysis/tokenizer/letter"
-	"github.com/blevesearch/bleve/v2/document"
 	"github.com/blevesearch/bleve/v2/search"
-	index "github.com/blevesearch/bleve_index_api"
-	"github.com/mitchellh/mapstructure"
 	"github.com/pkg/errors"
 )
 
 const ResultsPerPage = 20
+const indexFilename = "index.bleve"
 
-type DocumentMatch[T options.NixOption] struct {
+type DocumentMatch struct {
 	search.DocumentMatch
-	Data T
+	Data options.NixOption
 }
 
-type Result[T options.NixOption] struct {
+type Result struct {
 	*bleve.SearchResult
-	Hits []DocumentMatch[T]
+	Hits []DocumentMatch
 }
 
-type Index[T options.NixOption] struct {
+type ReadIndex struct {
 	index bleve.Index
 }
 
-func New[T options.NixOption](kind string) (*Index[T], error) {
-	var err error
-	bleve.SetLog(log.Default())
+func Open(dir string) (*ReadIndex, error) {
+	indexPath := path.Join(dir, indexFilename)
 
-	indexMapping := bleve.NewIndexMapping()
-
-	textFieldMapping := bleve.NewTextFieldMapping()
-	textFieldMapping.Store = false
-
-	descriptionFieldMapping := bleve.NewTextFieldMapping()
-	descriptionFieldMapping.Store = false
-	descriptionFieldMapping.Analyzer = "web"
-
-	err = indexMapping.AddCustomAnalyzer("option_name", map[string]interface{}{
-		"type":      custom.Name,
-		"tokenizer": letter.Name,
-		"token_filters": []string{
-			camelcase.Name,
-		},
-	})
+	idx, err := bleve.Open(indexPath)
 	if err != nil {
-		return nil, errors.WithMessage(err, "could not add custom analyser")
+		return nil, errors.WithMessagef(err, "unable to open index at path %s", indexPath)
 	}
-	nameMapping := bleve.NewTextFieldMapping()
-	nameMapping.Analyzer = "option_name"
-	nameMapping.IncludeTermVectors = true
-	nameMapping.Store = false
-
-	nixValueMapping := bleve.NewDocumentStaticMapping()
-	nixValueMapping.AddFieldMappingsAt("Text", textFieldMapping)
-	nixValueMapping.AddFieldMappingsAt("Markdown", textFieldMapping)
-
-	optionMapping := bleve.NewDocumentStaticMapping()
-
-	optionMapping.AddFieldMappingsAt("Option", nameMapping)
-	optionMapping.AddFieldMappingsAt("Loc", bleve.NewKeywordFieldMapping())
-	optionMapping.AddFieldMappingsAt("RelatedPackages", textFieldMapping)
-	optionMapping.AddFieldMappingsAt("Description", textFieldMapping)
-
-	optionMapping.AddSubDocumentMapping("Default", nixValueMapping)
-	optionMapping.AddSubDocumentMapping("Example", nixValueMapping)
-
-	indexMapping.AddDocumentMapping("option", optionMapping)
-
-	idx, err := bleve.NewMemOnly(indexMapping)
-	// index, err = bleve.New(path.Join(cfg.DataPath, const indexFilename = "index.bleve"), indexMapping)
-
-	if err != nil {
-		return nil, errors.WithMessage(err, "error opening index")
-	}
-	batch := idx.NewBatch()
-
-	jsonFile, err := os.Open(path.Join("data", "processed", kind+".json"))
-	if err != nil {
-		return nil, errors.WithMessage(err, "error opening json file")
-	}
-
-	dec := jstream.NewDecoder(jsonFile, 1)
-	var opt options.NixOption
-	ms, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
-		ErrorUnused: true,
-		ZeroFields:  true,
-		Result:      &opt,
-	})
-	if err != nil {
-		return nil, errors.WithMessage(err, "could not create struct decoder")
-	}
-	for mv := range dec.Stream() {
-		opt = options.NixOption{}
-		orig := mv.Value.(map[string]interface{})
-		err := ms.Decode(orig) // stores in opt
-		if err != nil {
-			return nil, errors.WithMessagef(err, "could not decode value: %+v", orig)
-		}
 
-		doc := document.NewDocument(opt.Option)
-		err = indexMapping.MapDocument(doc, opt)
-		if err != nil {
-			return nil, errors.WithMessagef(err, "could not map document for option: %s", opt.Option)
-		}
+	return &ReadIndex{
+		idx,
+	}, nil
+}
 
-		var data bytes.Buffer
-		enc := gob.NewEncoder(&data)
-		err = enc.Encode(opt)
-		if err != nil {
-			return nil, errors.WithMessage(err, "could not store option in search index")
-		}
-		field := document.NewTextFieldWithIndexingOptions("data", nil, data.Bytes(), index.StoreField)
-		newDoc := doc.AddField(field)
+func (index *ReadIndex) GetSource(ctx context.Context, name string) (*bleve.SearchResult, error) {
+	query := bleve.NewTermQuery(name)
+	query.SetField("Source")
+	search := bleve.NewSearchRequest(query)
 
-		err = batch.IndexAdvanced(newDoc)
+	result, err := index.index.SearchInContext(ctx, search)
 
+	select {
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	default:
 		if err != nil {
-			return nil, errors.WithMessagef(err, "could not index option %s", opt.Option)
+			return nil, errors.WithMessagef(err, "failed to execute search to find source %s in index", name)
 		}
 	}
-	err = idx.Batch(batch)
-	if err != nil {
-		return nil, errors.WithMessage(err, "failed to run batch index operation")
-	}
 
-	return &Index[T]{
-		idx,
-	}, nil
+	return result, nil
 }
 
-func (index *Index[T]) Search(ctx context.Context, keyword string, from uint64) (*Result[T], error) {
-	query := bleve.NewMatchQuery(keyword)
-	query.Analyzer = "option_name"
+func (index *ReadIndex) Search(ctx context.Context, source string, keyword string, from uint64) (*Result, error) {
+	sourceQuery := bleve.NewTermQuery(source)
+	userQuery := bleve.NewMatchQuery(keyword)
+	userQuery.Analyzer = "option_name"
+
+	query := bleve.NewConjunctionQuery(sourceQuery, userQuery)
+
 	search := bleve.NewSearchRequest(query)
 	search.Size = ResultsPerPage
-	search.Fields = []string{"data"}
+	search.Fields = []string{"_data"}
 	search.Explain = true
 
 	if from != 0 {
@@ -164,10 +86,10 @@ func (index *Index[T]) Search(ctx context.Context, keyword string, from uint64)
 			return nil, errors.WithMessage(err, "failed to execute search query")
 		}
 
-		results := make([]DocumentMatch[T], min(ResultsPerPage, bleveResult.Total))
+		results := make([]DocumentMatch, min(ResultsPerPage, bleveResult.Total))
 		var buf bytes.Buffer
 		for i, result := range bleveResult.Hits {
-			_, err = buf.WriteString(result.Fields["data"].(string))
+			_, err = buf.WriteString(result.Fields["_data"].(string))
 			if err != nil {
 				return nil, errors.WithMessage(err, "error fetching result data")
 			}
@@ -178,7 +100,7 @@ func (index *Index[T]) Search(ctx context.Context, keyword string, from uint64)
 			buf.Reset()
 		}
 
-		return &Result[T]{
+		return &Result{
 			SearchResult: bleveResult,
 			Hits:         results,
 		}, nil
generated by cgit-pink 1.4.1 (git 2.36.1) at 2024-11-21 21:27:29 +0000