Elasticsearch Embedding

Install CLIP

pip install sentence-transformers

Load Pretrain Text Model

text_model = SentenceTransformer('sentence-transformers/clip-ViT-B-32-multilingual-v1')

Load Pretrain Image Model

img_model = SentenceTransformer('clip-ViT-B-32')

Text embedding

corpus = [
    'con chó trong tuyết',
    'chú mèo',
    'cây dừa cạnh bờ biển'
]
vec_text = text_model.encode(corpus)

Image Embedding

from PIL import Image
images = []
for f in ['dog.jpg', 'coconut.jpg', 'cat.jpg']:
    img = Image.open(path.join('./images', f))
    images.append(img)
vec_img = img_model.encode(images)

Create Schema product

productMapping = {
    "properties":{
        "title":{
            "type":"keyword"
        },
        "image":{
            "type":"text"
        },
        "titleVec":{
            "type":"dense_vector",
            "similarity": "cosine",
            "dims": 512
        },
        "imageVec":{
            "type":"dense_vector",
            "similarity": "cosine",
            "dims": 512
        }
    }
}

Elasticsearch create Index Schema

client = Elasticsearch('http://localhost:9200')
client.ping()
if client.indices.exists(index='product'):
    client.indices.delete(index='product')
client.indices.create(index='product', mappings=productMapping)

Insert data from to Index

for row in df.to_dict('records'):
    client.index(index = 'product', document=row)

Search knn

query = 'đại bàng'
q_vec = text_model.encode(query)
print(q_vec.shape)
dic = {
    "field": "titleVec",
    "query_vector": list(q_vec),
    "k": 3,
    "num_candidates": 512
}
client.knn_search(index='product', knn = dic, filter_path='hits.hits', source=['title', 'image'])