
    t5hl7                         d dl Z d dlZd dlZd dlZd dlmZmZmZmZ d dl	m	Z	 d dl
mZ d dlZd dlZd dlZd dlmZ d dlmZ d dlmZmZmZmZmZmZmZmZmZ d dlmZ d d	lmZ d d
lm Z   G d d      Z! G d d      Z"y)    N)ListDictAnyOptional)datetime)settings)AzureOpenAI)QdrantClient)	PointStructVectorParamsDistanceFieldConditionFilterRange
MatchValueMatchAny	MatchText)HTTPException)	Templates)Sessionc                   2    e Zd Zd ZdedefdZdedefdZy)EmbeddingServicec                 ~    t        dt        j                  t        j                        | _        d| _        d| _        y )Nz
2024-02-01)api_versionazure_endpointapi_keyEmbedLarge3   )r	   r   AZURE_OPENAI_API_ENDPOINTAZUREKEY_CREDENTIAL_API_KEYclient
deploymentembedding_dimensionsselfs    I/var/www/html/hubwallet-dev/src/marketing/vector_db_collection/service.py__init__zEmbeddingService.__init__   s4    !$#==88

 ($(!    textreturnc                    |rt        |t              sdg| j                  z  S | j                  |      }|sdg| j                  z  S 	 | j                  j
                  j                  |g| j                        }|j                  d   j                  S # t        $ r1}t        dt        |              dg| j                  z  cY d}~S d}~ww xY w)z-Generate embedding with robust error handling        )inputmodelr   zEmbedding generation failed: N)
isinstancestrr#   _clean_textr!   
embeddingscreater"   data	embedding	Exceptionprint)r%   r)   
clean_textresponsees        r&   generate_embeddingz#EmbeddingService.generate_embedding'   s    :dC0544444 %%d+
544444	5{{--44!loo 5 H ==#--- 	51#a&:;544444	5s   A
B 	C&CCCc                     dj                  |j                               }d}t        |      |kD  r|d| }|j                         S )z*Clean text before sending to embedding API @  N)joinsplitlenstrip)r%   r)   
max_lengths      r&   r1   zEmbeddingService._clean_text<   sC     xx

% 
t9z!$Dzz|r(   N)__name__
__module____qualname__r'   r0   listr;   r1    r(   r&   r   r      s-    )5s 5t 5*
 
 
r(   r   c                   6   e Zd Zd Zdedee   fdZdedefdZ	dded	e
ddfd
Zd	e
defdZdedededdfdZ	 	 	 ddedededee   dedefdZdeeeef      ded	e
defdZdedee   fdZdededee   fdZdededee   fdZdedefdZdedefdZy)QdrantServicec                     t               | _        d| _        t        t        j
                  t        j                  d      | _        y )Nr   T)urlr   prefer_grpc)r   embedding_servicer#   r
   r   QADERND_URLQDRANT_API_KEYr!   r$   s    r&   r'   zQdrantService.__init__K   s6    !1!3$(!"$$++
r(   r)   r*   c                 6   	 | j                   j                  |      }t        |      | j                  k7  r't	        dt        |              dg| j                  z  S |S # t
        $ r1}t	        dt        |              dg| j                  z  cY d}~S d}~ww xY w)zWrapper with additional loggingz/Warning: Invalid embedding dimension received: r,   zEmbedding service error: N)rN   r;   rA   r#   r7   r6   r0   )r%   r)   r5   r:   s       r&   _get_embeddingzQdrantService._get_embeddingV   s    	5..AA$GI9~!:!::GIGWXYut8888 	5-c!fX67544444	5s$   AA A 	B'&BBBr4   c           	      f   g }t        |t              r|S t        |t              r)|D ]"  }|j                  | j	                  |             $ nt        |t
              r|j                         D ][  }t        |t              r|j                  |       %t        |t
        t        f      s<|j                  | j	                  |             ] d|v r|d   j                         D ]  }t        |t
              s|j                  di       j                  d      dk(  s9|j                  di       j                  dd      }|s^|j                  t        j                  dd|              d	j                  |D cg c]$  }|r t        |t              r|j                         & c}      }|d
d S c c}w )z+Enhanced text extraction from template datalayerstyperesolvedName	TextLayerpropsr)    z<[^>]+>r=   Nr>   )r/   r0   rG   append_extract_search_textdictvaluesgetresubr?   rB   )	r%   r4   
text_partsitemvaluelayerr)   tr8   s	            r&   r[   z"QdrantService._extract_search_textb   s   
dC KdD! C!!$";";D"ABC d# HeS)%%e,d|4%%d&?&?&FG	H 4!(^224 PE!%. 99VR044^DS#(99Wb#9#=#=fb#ID# * 1 1"&&R2N OP XX)
Z3' GGI
 
 %4  	
s   9)F.Nnamedbc                 8   | j                   j                  |      s| j                   j                  |t        | j                  t
        j                        ddd       g d}|D ]  \  }}| j                  |d| |        |r| j                  ||       y	y	y	)
z/Create collection optimized for template search)sizedistance   r   )default_segment_numberindexing_threshold)vectors_configoptimizers_config))template_typekeyword)industryrq   )tagsrq   )design_stylerq   )color_themerq   z	metadata.)rg   rf   N)	r!   collection_existscreate_collectionr   r#   r   COSINE_create_field_index_mark_template_as_processed)r%   rf   rg   metadata_fieldsfield
field_types         r&   rw   zQdrantService.create_collection   s    {{,,T2KK))+22%__  /0*+# * O &5 !z((w' 00BT0B = 3r(   c                     |j                  t              j                  t        j                  |k(        j	                         }|r9|j
                  dk7  r)d|_        |j                          |j                  |       y y y )N   )queryr   filtertemplate_namefirstdata_processcommitrefresh)r%   rg   rf   templates       r&   rz   z)QdrantService._mark_template_as_processed   sc    88I&--i.E.E.MNTTV--2$%H!IIKJJx  38r(   
collectionr|   r}   c           	          	 | j                   j                  |||       t        j                  d| d|        y # t        $ r.}t        j
                  d| dt        |              Y d }~y d }~ww xY w)N)collection_name
field_namefield_schemazCreated index for z as zFailed to create index for : )r!   create_payload_indexlogginginfor6   warningr0   )r%   r   r|   r}   r:   s        r&   ry   z!QdrantService._create_field_index   sw    	MKK,, * ' - 
 LL-eWDEF 	MOO9%3q6(KLL	Ms   9< 	A3$A..A3r   limitfiltersscore_thresholdc                 L   | j                   j                  |      st        dd| d      | j                  |      }|r| j	                  |      nd }| j                   j                  |||||d      }||D 	cg c]  }	|	j                  dkD  r|	j                  j                  d|	j                        |	j                  |	j                  j                  di       |	j                  j                  d	d
      | j                  ||	j                        d c}	t        |      dS c c}	w )N  Collection '' not foundT)r   query_vectorquery_filterr   r   with_payloadg333333?original_id	json_datasearchable_textrY   )idscorer4   r   
highlights)r   resultstotal_results)r!   rv   r   rR   _parse_filterssearchr   payloadr^   r   _find_highlightsrA   )
r%   r   r   r   r   r   vectorr   r   rs
             r&   r   zQdrantService.search   s    {{,,Z8|J<{%KLL $$U+ 8?t**73D ++$$&%+ % 
  #3 aggck ))--qtt<WWIIMM+r:'(yy}}5F'K"&"7"7qyy"I 3 "%W
 
	3s   8BD!docsc                    | j                  ||       g }|D ]  }	 t        |d         }	 t        j                  t        j                  |      }| j                  |d         }| j                  |      }	t        t        |      |	||j                  d      ||j                  d      |j                  di       t        j                         j                         d      }
|j                  |
        t        d|       |sdddS 	 | j                   j#                  ||d      }dt%        |      |j&                  dS #  t        j
                         }Y xY w# t        $ r4}t        d|j                  d       d	t        |              Y d
}~zd
}~ww xY w# t        $ r/}t        dt        |              dt        |      dcY d
}~S d
}~ww xY w)z>Insert documents with specific fields from database in payload)rg   r   template_metadatatemplate_urlr   )r   r   template_idr   r   inserted_at)r   r   r   zFailed to process document r   NpointsFzNo valid documents to insert)successerrorT)r   r   wait)r   inserted_countoperation_idzQdrant upsert failed: )rw   r0   uuiduuid5NAMESPACE_DNSuuid4r[   rR   r   r^   r   utcnow	isoformatrZ   r6   r7   r!   upsertrA   r   )r%   r   r   rg   r   docr   doc_uuidsearch_textr5   pointr:   operation_results                r&   insert_documentszQdrantService.insert_documents   s   zb1 	C!#d)n,#zz$*<*<kJH
 #77<O8PQ //<	 $8}$+6(+(?'2),)A-0WW5H"-M'/'8'B'B'D e$3	: 	hv$/MNN	7#{{11 *  2    "%f+ 0 = = E,#zz|H(  3CGGDM?"SVHMN$  	7*3q6(34$s1v66	7sM   E$D>B"E6F >EE	F#)FF	G#$GGGc                    g }d|v r|d   }n|g}|D ]=  }d|v rJ|d   j                  dd      }|j                  di       j                  d      xs |j                  d      }n%t        t        |j	                                     \  }}d| }t        |t              rTd|v r*|j                  t        |t        |d         	             |j                  t        |t        di |
             t        |t              r(|j                  t        |t        |      	             |j                  t        |t        |      	             @ |rt        |      S dS )z*Convert API filter format to Qdrant Filtermustkeyz
json_data.rY   matchrc   any)r   )r   r   )r   range)rc   )r   NrH   )replacer^   nextiteritemsr/   r\   rZ   r   r   r   rG   r   r   )r%   r   
conditionsfilter_itemsrb   r|   rc   r   s           r&   r   zQdrantService._parse_filters  sA   
 W"6?L#9L  	aD}U++L"="-11':Odhhw>O#D$67u%eW-J%&E>%%n8X]^cXdKe&fg %%n5>SX>&Z[E4(!!.ZxTYGZ"[\!!.ZzX]G^"_`'	a* +5v:&>$>r(   r   c                   	 t        |j                         j                               }|j                  dd      j                         }|j                  di       }g }|D ]'  }t	        |      dkD  s||v s|j                  |       ) |j                         D ]A  \  }	t        	t              st        	fd|D              s,|j                  | d	        C |dd S )	zIdentify matching text snippetsr   rY   r      c              3   B   K   | ]  }|j                         v   y wN)lower).0termrc   s     r&   	<genexpr>z1QdrantService._find_highlights.<locals>.<genexpr>G  s     -\dekkm.C-\s   :N   )
setr   r@   r^   rA   rZ   r   r/   r0   r   )
r%   r   r   query_termsr)   fieldsr   r   r|   rc   s
            @r&   r   zQdrantService._find_highlights:  s    %++---/0{{,b1779["-
 	(D4y1}!!$'	(
 #LLN 	6LE5%%#-\P[-\*\!!UG1UG"45	6 "1~r(   c                 N   g }|j                  di       }|j                  dg       }t        |j                         j                               }|D ]Q  }t	        |j                  |d            j                         t        fd|D              sA|j                  |       S |S )z
        Identify which fields likely matched the query
        
        Args:
            query: Original search query
            payload: Document payload
            
        Returns:
            List of field names that likely matched
        r   searchable_fieldsrY   c              3   D   K   | ]  }t        |      d kD  s|v   yw)r   N)rA   )r   r   field_values     r&   r   z5QdrantService._find_matched_fields.<locals>.<genexpr>_  s      P4#d)a-4;&Ps    	 )r^   r   r   r@   r0   r   rZ   )	r%   r   r   matched_fieldsr   search_fieldsr   r|   r   s	           @r&   _find_matched_fieldsz"QdrantService._find_matched_fieldsL  s     KKR0	$7< %++---/0" 	-EimmE267==?KP;PP%%e,	-
 r(   c                    | j                   j                  |      st        dd| d      | j                   j                  |      }|j                  |j
                  |j                  |j                  j                         dS )zGet collection informationr   r   r   )statusvectors_countpoints_countconfig)	r!   rv   r   get_collectionr   r   r   r   r\   )r%   rf   r   s      r&   collection_infozQdrantService.collection_infoe  sq    {{,,T2|D6%EFF{{))$/kk!// --kk&&(	
 	
r(   c                     | j                   j                  |      st        dd| d      | j                   j                  |       dd| ddS )zDelete a collectionr   r   r   Tz	' deleted)r   message)r!   rv   r   delete_collection)r%   rf   s     r&   r   zQdrantService.delete_collectionq  sP    {{,,T2|D6%EFF%%d+l4&	,JKKr(   r   )r   Ng333333?)rD   rE   rF   r'   r0   r   floatrR   r   r[   r   rw   rz   ry   intr   r   r   r   r   r   r   r   r   r   rH   r(   r&   rJ   rJ   J   s   

53 
54; 
5!! !! !!F!Cc !Cw !C$ !CF!g !S !	Mc 	M# 	M3 	MSW 	M  "&!$%% % 	%
 $% % 
%L37T$sCx.%9 37s 37PW 37\` 37n?d ?x/? ?Dc D T#Y $#  c 2

C 

D 

Lc Ld Lr(   rJ   )#osr   jsonr   typingr   r   r   r   r   src.utils.settingsr   r_   openaiqdrant_clientr	   r
   qdrant_client.modelsr   r   r   r   r   r   r   r   r   fastapir   "src.menu_design.apps.editor.modelsr   sqlalchemy.ormr   r   rJ   rH   r(   r&   <module>r      sY      , ,  ' 	    &
 
 
 " 8 "
) )ZlL lLr(   