o !6i@s|ddlZddlZddlZddlZddlmZddlmZddlmZm Z m Z m Z m Z m Z ddlZddlZddlmZddlZddlZddlZddlZddlZddlmZddlmZddlZddlZddlZ ddl!Z"ddl#m$Z$m%Z%ddl&Z'ddl(Z(ddl)Z*dd l+m,Z,e,e-d Z.e-d Z/e-d Z0ej1ej2d de3e4Z5ddddgddddddgdddddgddddddgddddddgddddddgdddd ddgd!ddd"ddgd#ddd$ddgd%ddd&d'dgd(ddd)d'dgd*ddd+d'dgd,ddd-d.dgd/dd0d1d2dgd3ddd4d5dgd6dd7Z6Gd8d9d9Z7Gd:d;d;Z8GdkrZded?edefd@dAZdOd1ededCedefdDdEZdFe j!ddfdGdHZ"dIedefdJdKZ#dS)P ApiClientz,Handles all API calls to the Node.js servicecCs"|d|_||_|dd|_dS)zInitialize API client/application/json)x-internal-api-keyz Content-TypeN)rstripbase_urlapi_keyheaders)selfrArBr:r:r;__init__s  zApiClient.__init__ tenant_idreturncC6|jd|}tj||jd}|||dS)zGet tenant by ID /tenants/rCdatarArequestsgetrC_check_responsejsonrDrFurlresponser:r:r; get_tenant  zApiClient.get_tenantN company_idcCsX|dur|jd|d}n |jd|d|}tj||jd}|||dS)z,Get formatted company information for tenantNrIz /company-infoz/company-info?company_id=rJrKrL)rDrFrVrRrSr:r:r;get_company_infos   zApiClient.get_company_infocCrH)z4Get raw analysis of company's document from databasez/rawAnalysisById/rJrKrL)rDrVrRrSr:r:r;get_company_analysis_datarUz#ApiClient.get_company_analysis_data tender_idcCs2|jd|d|}tj||jd}|||d}d|vrtdt|dtd|dt |dt rg}|dD]C}t |t rZd|vrZ| t |dqEt |t re| |qEtd t|d |z | t |WqEtd |YqE||d<td |d|S) zGet tender by ID and tenant /tenders/ ?tenant_id=rJrKoriginalDocumentsz'Original documents in tender response: zDocument IDs: _idzUnexpected document ID format:  - z)Could not convert document ID to string: zConverted document IDs: )rArMrNrCrOrPloggerinfotype isinstancelistdictappendstrwarningerror)rDrYrFrRrS tender_data string_docsdocr:r:r; get_tenders,     zApiClient.get_tender update_datacC8|jd|}tj|||jd}|||dS)zUpdate tender with new datarZrPrCrKrArMputrCrOrP)rDrYrmrRrSr:r:r; update_tender  zApiClient.update_tender document_idsc Cs|jd}g}|D]4}|durq t|tr$d|vr$|t|dq t|tr7d|vr7|t|dq |t|q td|tj|d|i|j d}|j dkrt d |j z| }t d |Wnt d |j dd Y||| d S)zGet documents by IDsz/documents/get-manyNr]z$oidzRequesting documents with IDs: rtrozError getting documents: HTTP Error details: Raw response: rK)rArbrdrerfr_debugrMpostrC status_coderhrPtextrO)rDrtrR string_idsdoc_idrS error_infor:r:r; get_documentss*    zApiClient.get_documents document_datacC4|jd}tj|||jd}|||dS)zCreate a new document recordz /documentsrorKrArMrzrCrOrP)rDrrRrSr:r:r;create_document   zApiClient.create_document document_idc Cst|dddd}td||jd|d|}tj|d|jid}|jd krWt d |jz| }t d |Wnt d |j d dY| ||j S)zGet document content"'z!Getting content for document ID: z /documents/z/content?tenant_id=r?rJruz%Error getting document content: HTTP rvrwNrx)rfreplacer_r`rArMrNrBr{rhrPr|rOcontent)rDrrFr~rRrSrr:r:r;get_document_contents  zApiClient.get_document_content analysis_datacCr)zCreate a new analysis recordz /analysesrorKr)rDrrRrSr:r:r;create_analysis4rzApiClient.create_analysis analysis_idcCrH)zGet analysis by ID /analyses/rJrKrL)rDrrRrSr:r:r; get_analysis;rUzApiClient.get_analysiscCrn)zUpdate analysis with new datarrorKrp)rDrrmrRrSr:r:r;update_analysisBrszApiClient.update_analysiscCs>|jd|d|}tj|||jd}|||dS)z'Update company info with extracted dataz/companyDetails/r=rorKrp)rDrVrFrmrRrSr:r:r;update_company_infoIs  zApiClient.update_company_infobid_datacCr)z"Create a new bid generation recordz/bid-generationsrorKr)rDrrRrSr:r:r;create_bid_generationPrzApiClient.create_bid_generationbid_generation_idcCs<|jd|d|}tj||jd}|||dS)z#Get bid generation by ID and tenant/bid-generations/r[rJrKrL)rDrrFrRrSr:r:r;get_bid_generationWs  zApiClient.get_bid_generationcCrn)z#Update bid generation with new datarrorKrp)rDrrmrRrSr:r:r;update_bid_generation^rszApiClient.update_bid_generationF file_contents3_key content_typeencryptc Csj|jd}dtj|||fi}|||t|d}d|ji} tj|||| d} | | | dS)zUpload file to S3z/storage/uploadfile)rFkeyrrr?)filesrKrCrK) rAospathbasenamerflowerrBrMrzrOrP) rDrrrrFrrRrrKrCrSr:r:r; upload_filees     zApiClient.upload_fileTdecryptcCsH|jd|d|dt|}tj|d|jid}|||jS)zDownload file from S3z/storage/download?tenant_id=&key=z &decrypt=r?rJ)rArfrrMrNrBrOr)rDrrFrrRrSr:r:r; download_file{s $ zApiClient.download_filecCs8|jd|d}tj||jd}|||dS)z#List company documents for a tenant/storage/company-documents/z/listrJrKrLrQr:r:r;list_company_documentss  z ApiClient.list_company_documents download_dirc Cs|jd|d}tj|d|jid}||tj|d}t|d}| |j Wdn1s5wYddl }| |d  }| |Wdn1sTwYt|d} t|D] \} } } | t| 7} qe| S) z+Download all company documents for a tenantrz /downloadr?rJzcompany_documents.zipwbNrr)rArMrNrBrOrrjoinopenwriterzipfileZipFile extractallremovewalklen) rDrFrrRrSzip_pathfrzip_ref file_count_rr:r:r;download_company_documentss$    z$ApiClient.download_company_documentslocal_docs_pathc Cs|jd|d}g}t|D],\}}}|D]$}|ds#|dr$qtj||} |d|t| d| |ffqqd|j i} t j ||| d} |D] \}} | d  qL|| | d S) z(Upload company documents from local pathrz/upload..tmprrbr?)rrCrK)rArr startswithendswithrrrer_get_content_typerBrMrzcloserOrP) rDrFrrRrrootr filenamesfilename file_pathrCrS file_tupler:r:r;upload_company_documentss"    z"ApiClient.upload_company_documents expires_incCsF|jd|d|d|}tj||jd}|||ddS)zGet signed URL for S3 objectz/storage/signed-url?tenant_id=rz &expires_in=rJrKrRrL)rDrrFrrRrSr:r:r;get_signed_urls zApiClient.get_signed_urlrScCs||jdkr).pdf.html.txt.docx.xlsx.jsonapplication/octet-streamrrsplitextrrNrDrext content_typesr:r:r;rs zApiClient._get_content_typeN)F)T)r)$r,r-r.r/rErfrrTrWrXrlrrrrrbytesrrrrrrrrboolrrrintrrrrMResponserOrr:r:r:r;r<sB  #&  r<c@seZdZdZd}defddZd~deded efd d Zd eded eeeffd dZ d ededed efddZ d edededed df ddZ d ededed eeeffddZ d~dede eeefded eeeffddZd~dededed eeeffddZdedede eeefd eeeffddZd~dedede eeefded eeeff dd Z d~deded!ed"eded eeeff d#d$Zded%ed eeeffd&d'Zdeded eeeffd(d)Zdedefd*d+Zd~deded ed,ed-ed ef d.d/Z  0ddeded1e ed2e ed3ed eeeff d4d5Z 6 0dded!eded7ed8ed eeeff d9d:Z 6 0 ddeded7ed8ed2e ed eeeff d;d<Z 0 0dd eded3ed=ed eeeff d>d?Zd@eeefdAed dfdBdCZdDed eeeffdEdFZdGed efdHdIZdJedededKed e ef dLdMZ dNe edededKedOe ed e ef dPdQZ!defdRdSZ"dAedTefdUdVZ#dedWedXefdYdZZ$d[ed efd\d]Z%d[ed^ed efd_d`Z&dadbZ'dced efdddeZ( d~dedfedgeded e ef dhdiZ)djedked efdldmZ*d[ed^edged efdndoZ+d[ed efdpdqZ,d[ed efdrdsZ-dtedDed efdudvZ.dted dfdwdxZ/dted efdydzZ0dted efd{d|Z1dS)MinaionsRFPServicez1Main service for RFP processing using Minaions AI config.json config_pathc Cszt|d}t||_Wdn1swYWnty2}z td|d}~wwtj|jddd|_ |j di dd |_ t j |j d d |j d i d d}|j d i dt j d}|sstdt|||_t|jdd|_d|_d|_d|_dS)z)Initialize the service with configurationrNzFailed to load configuration: anthropicrB)rB file_storagebase_dirz/tmp/rfp_serviceTexist_oknodejs_serviceapi_urlz"http://localhost:5000/internal-apiINTERNAL_API_KEYzgInternal API key is required. Please set it in config.json or as INTERNAL_API_KEY environment variable.)rrPloadconfigrr_rhrClientclientrNrrmakedirsenviron ValueErrorr< api_clientr extractorcurrent_tenant_idcurrent_process_idcurrent_process_type)rDrre api_base_urlrBr:r:r;rEs,   zMinaionsRFPService.__init__NrFrVrGc, Cs` z0|j||}|di}|di}|jd|ikri}|dg}|D]"}|dd} |dd} |dd} | rL| sD| rL| rH| n| || <q*d } | d 7} | d 7} | d 7} |d rj| d|d d7} |dry| d|dd7} |dr| d|dd7} |dg} t| dD]\}}|r| d|d|d7} q|dr| d|dd7} |dr| d|dd7} |dr| d|dd7} |dr| d|dd7} |d r| d!|d d7} |d"r| d#|d"d7} | d7} | d$7} | d%7} |d&r| d'|d&d7} |d(r+| d)|d(d7} |d*r;| d+|d*d7} | d7} | d,7} | d-7} |d.rW| d/|d.d7} |d0rg| d1|d0d7} |d2rw| d3|d2d7} |d4r| d5|d4d7} |d6r| d7|d6d7} |d8r| d9|d8d7} | d7} |d:g}|r| d;7} | d%7} t|dD]Q\}}|dr| d<|d=7} | d>|dd7} |d?r| d@|d?d7} |dAr| dB|dAd7} |dCr | dD|dCd7} | d7} q|dEi}|r| dF7} | d 7} |dGg}|D]'}|dHrR|dIrR|dJd}| dK|dHd|dL|dId7} q,|dMg}|D]}|dNrn| dO|dNd7} q\|dPg}|D]}|dQr| dR|dQd7} qx|dSg}|D]}|dTr| dU|dTd7} q| d7} |dVg}|rP| dW7} | dX7} t|dD]\}}|dYs|dZrN| d[|d=7} |dYr| d\|dYd7} |d]r| d^|d]d7} |dZr | d_|dZd7} |d`r| da|d`d7} |dbr*| dc|dbd7} |ddr:| de|ddd7} |dfrJ| dg|dfd7} | d7} q|dhg}|r{| di7} | dj7} |D]}|dkru| dl|dkd7} qc| d7} | dm7} |jdnt| do|jdp| | WSi}|dg}|D]'}|dd} |dd} |dd} | r| s| r| r| n| || <qd } | d 7} | d 7} | d 7} |d r| d|d d7} |dr| d|dd7} |dr| d|dd7} |dg} t| dD]\}}|r$| d|d|d7} q|dr6| d|dd7} |drF| d|dd7} |drV| d|dd7} |drf| d|dd7} |d rv| d!|d d7} |d"r| d#|d"d7} |dqr| dr|dqd7} | d7} t} | ds7} | d 7} |D]y\}!} | dtt j | duidvdw| dk| dxf}"|"| vr̐q| |"| dtdy}#| dkdy}$| dxdy}%| dui}&| dz|#d7} | d{|$d7} | d||%d7} |&r| d}7} |&D]\}'}(| d~|'d|(d7} q | d7} q|jdt| d| WSt y})zq|j d|)z\|j||}|di}|dg}i}*|D]}|dd} |dd} | rs| rs| |*| <q[d|*d dd}+|+d|*ddd7}+|+d|*ddd7}+|+WWYd})~)SYWYd})~)dSd})~)ww)z Get and format company information from API (structured data only) Args: tenant_id: Tenant ID Returns: str: Formatted company information from structured data company_info raw_analysisz-Retrieved company info (structured data only)detailsnamervaluerz*COMPANY INFORMATION FOR TENDER SUBMISSION z>============================================================ zBASIC COMPANY DETAILS: z---------------------- rzCompany Name:  rz Entity Type: registeredAddresszRegistered Address: branchOfficeAddressrzBranch Office Address rrz Website: officePhoneNumberzPhone: officeEmailzEmail: officeFaxzFax: authorizedPersonToSignDocszAuthorized Signatory: authorizedPersonDesignationz Designation: zEMPLOYEE INFORMATION: z--------------------- totalEmployeeszTotal Employees: technicalStaffzTechnical Staff: rzEmployee Details Document: zLEGAL & REGISTRATION DETAILS: z---------------------------- r z$Company Incorporation Number (CIN): r!z Permanent Account Number (PAN): r"zGoods and Services Tax (GST): r#z9Micro, Small and Medium Enterprises (MSME) Registration: typeOfMsmez MSME Type: r$zVDepartment for Promotion of Industry and Internal Trade (DPIIT) Startup Registration: directorDetailszDIRECTOR INFORMATION: z Director z: z Name: addressz Address: panz" Permanent Account Number (PAN): dinz( Director Identification Number (DIN): financialDetailszFINANCIAL INFORMATION: turnoverDetails turnoverYearturnover turnoverUnitzAnnual Turnover  itrDetailsitrYearzIncome Tax Return (ITR) Year: ca_certificate_detailsca_certificate_descriptionz'Chartered Accountant (CA) Certificate: balanceSheetDetailsauditedBalanceSheetYearBalance Sheet Year: pastExperienceDetailszPAST EXPERIENCE: z---------------- customerprojectzProject z Client Name: clientLocationz Client Location: z Project Title: projectValuez Project Value: projectScopez Project Scope: projectStartDatez Start Date: projectEndDatez End Date: r*zCERTIFICATIONS: z-------------- descriptionz- z============================================================= z,Successfully formatted company information (z characters)zCompany information: aboutz!About Company (Additional Info): zDOCUMENT ANALYSIS INFORMATION doc_typekey_infoT) sort_keysrN/AzDocument Type: z Description: z File Path: zKey Information:  - z>------------------------------------------------------------ z Formatted z# unique documents from raw_analysisz&Error formatting company information: z Not specifiedz Contact: Nz!Company information not available)rrWrNprocess_loggerr` enumeratersetitemsrPdumpsaddrrh),rDrFrV company_datarr details_info details_listitemrrfile_urlformatted_infobranch_addressesirr(directorfinancial_detailsturnover_detailsrunit itr_detailsitrca_cert_detailsca_certbalance_sheet_details balance_sheetr)r certificatescertseenridentityr/r-rr0kvr basic_info fallback_infor:r:r;_format_company_information s                                     $                \                           z.MinaionsRFPService._format_company_informationrYc# Csz|||d|j|ddtddii|jd|d||j||}|s5td||jd ||j|}|sMtd |t j |j d |}t j |d d g}|jd|d||dg}dd|D}|jd||std|j|ddtddddddiid|ddgddWS|jdt|d|j|}|jdt|dg} |D]} zt| d d!} | d"d#| } t| d$d!} t| d%d!}| d&kr|d'kr|jd(| d)| d*td+| |j| |}|jd,| d-t|d.t j || }t|d/ }||Wd0n 1sBwY|jd1|| d2r||jd3| t|}|r|| ||jd4t|d5| zt |Wnty}z|jd6|d-|WYd0}~nd0}~wwWqty}z|jjd7| d d-|d d8WYd0}~qd0}~wwtt | }|jd9t|d:|rtj!|||jd;}t"|t#|}|D]}zzt j $|}t|d< }|%}Wd0n 1swY|dt|}t&'d=d>|}d?|d@|dA|}|(|}|j)||||}||dBd'dC|||dD|dEdF}|j*|} t| d } |+| |t|dG|jdH|Wqty}z|jdI|d-|WYd0}~qd0}~wwdJd|D}dKd|dgD} | |}!ddL|iidtdMt|t|dNdidO}"|j||"|j|ddtdt|t|dNdiizt,-|WnYd|t|t||dPt|dQdWSty?}z6tdRt|z|j|ddtdSdTt|idiiWnYdTdRt|dUWYd0}~Sd0}~ww)VzKExtract links from existing RFP documents and download additional documentsget_rfp_documents$setlogsget_rfp_documents_started timestampactionzGetting tender z for tenant Tender not found: zGetting tenant zTenant not found: rfp_docs_Trz,Getting additional RFP documents for tender bidNumberr\cSsg|]}|rt|qSr:rf.0r~r:r:r; wsz8MinaionsRFPService.get_rfp_documents..zDocument IDs from tender: zNo document IDs found in tender$pushget_rfp_documents_completedrz'No documents found in tender to process) links_founddocuments_downloadedrrZr[rsuccess)statusrYrerfdownloaded_documentsrzGetting documentsz Retrieved r]rrdoc_rar bid_documenttenderzProcessing document:  (ID: )zGetting content for document zReceived content for document rz bytesrNSaved document to rzExtracting links from PDF: z Extracted z links from zCould not delete temp file zError processing document )exc_infoFound z" unique links across all documentsr4r[\/\\]rtenders/r=z/linked/rfps3extracted_link)tenderIdr^sourceparentTenderIdtenantrrar storageTypestorageDetailsmetadata)rrsizezSaved linked document: zError saving linked document cSsg|]}|dqS)rr:rarkr:r:r;rbcSg|]}t|qSr:r_r`r:r:r;rbrz$eachlinked_documents_added)rerf $addToSetrczSuccessfully downloaded z additional RFP documentszError getting RFP documents: get_rfp_documents_failedrh)rir)._setup_process_contextrrrtimer4r`rlrrTrrrrrrNr_rgrrrfrrrrrtaextract_links_from_pdfextendrrrhrcr6download_linked_documentsremove_duplicate_pdfslist_files_in_directoryrreadresubrrrreshutilrmtree)#rDrYrFrnr~processing_dirrjrt documents all_linksrkr~doc_namer/ doc_category doc_content temp_pathrlinks cleanup_errr unique_linksdownloaded_filesrrr bid_numberrrstorage_detailsdocument_record new_doc_idsexisting_doc_ids all_doc_idsrmr:r:r;rUQsj         $.         &      z$MinaionsRFPService.get_rfp_documentsc Csz<|j||}|std||||dddditdddid gd }|j|}t|d }td ||WStyQ}z t d |d}~ww)zGCreate a placeholder analysis record immediately and return analysis_idr\analysis_estimation estimatingrINRanalysis_estimation_startedrzCost estimation in progressrg r~rncompanyrari estimatedCostcurrencyanalysisMetricsrWr]z%Created placeholder analysis record: z%Error creating analysis placeholder: N) rrlrrrrfr_r`rrh) rDrYrFrVrnanalysis_recordanalysisrrr:r:r;create_analysis_placeholderHs6  z.MinaionsRFPService.create_analysis_placeholderrc# Cs`zV|||d|jdtd||j||}|s'td|dd|dgD}|j|}t j |j d|}t j |d } t j | d d t j | d } t j | d } d|d} |j| |d} t| dkr| dkrt| d }|| Wdn1swYnLg}|D]>}z|dd|d}|di}||dWqty}z|jd|dd|WYd}~qd}~wwtj|||jd}d|d} |j| |d} t| dkr| dkrt| d }|| Wdn 1swYi}d}d}|jdt|| |jd d!|jd}|j|}t j | d }|||}|jd"||j|}|jd#|d$||}|jd%|d&d'}t| d( }|}Wdn 1s~wYd|d} d)}|j|| ||}|jd*d'}t| d( }|}Wdn 1swYd|d} d+}|j|| ||}|jd,td-||}t j |rt!"||jd.|d/n |jd.|d0d1}g}|d2krd3}|d4|d5krd6}|d7t|d8kr!|d99}|d:t#||d;} d<| |t|||||d=d>}!|j$||!|jd?|jd@|tdA| WdSty}zKtdB|zdCt%%dDdEt&|idFgdG}!|j$||!Wnty}"ztdH|"WYd}"~"n d}"~"wwWYd}~dSWYd}~dSd}~ww)IzMProcess analysis cost estimation in background and update the analysis recordrz;Analyzing document complexity and preparing for analysis...z5[INTERNAL] Starting cost estimation for analysis_id: r\cSrr:r_r`r:r:r;rbzrzRMinaionsRFPService.process_analysis_cost_estimation_background..r\analysis_cost_analysis_cost_outputTr merged.txt doc_text.jsontender_analysis//doc_text.jsonFrfalserNrrlr]rrRFailed to get document URL rrt /merged.txt{Gzt?KAttempting document extraction with external extractor for cost analysis...rrBCompany Info: Company Info length:  wordsTotal word count is  Uploading files to S3 now.rrrUploaded merged.txt file to S3r>!Uploaded doc_text.json file to S3 Directory '' deleted successfully.' does not exist.?'333333?Large document sizea?Very large document size皙?Multiple documents estimated wordCount documentCountcomplexityFactorcomplexityIndicatorsbaseCost costPerWord)rirr(Analysis complexity assessment completedTotal words analyzed: [INTERNAL] Estimated cost: ₹z.Error in background analysis cost estimation: estimation_failedanalysis_estimation_failedrhrg)rirW-Failed to update analysis with error status: )'rr4r`r_rrlrrNrrrrrrrrrrrerrhrrr rrcount_total_wordsrT count_wordsrrmaxexistsrrroundrrrf)#rDrYrFrVrrnrtrtemp_dir output_dir out_txt_file out_json_filerrrrrkrrrrdocuments_texttotal_word_countbase_cost_per_wordmerged_file_pathrcompany_info_word_countrr base_costcomplexity_factorcomplexity_indicatorsestimated_costrm update_errorr:r:r;+process_analysis_cost_estimation_backgroundms      *                       z>MinaionsRFPService.process_analysis_cost_estimation_backgroundc"Cs|||d|d|j||}|std|dd|dgD}|j|}tj|j d|}tj|d}tj |d d tj|d } tj|d } d |d} |j | |d} t | dkr| dkrt | d } | | Wdn1swYnLg}|D]>}z|dd|d}|di}||dWqty}z|jd|dd|WYd}~qd}~wwtj|||jd}d |d} |j | |d} t | dkr| dkrt | d } | | Wdn 1swYi}d}d}|jdt|||jdd |jd}|j|}|||}|jd!||j|}|jd"|d#||}|jd$|d%d&}t | d' } | }Wdn 1skwYd |d} d(}|j|| ||}|jd)d&}t | d' } | }Wdn 1swYd |d} d*}|j|| ||}|jd+td,||}tj|rt !||jd-|d.n |jd-|d/d0}g}|d1krd2}|d3|d4krd5}|d6t |d7kr|d89}|d9t"||d:}|||dd;|d<|t |||||d=t##d>||t |d?d@gdA }|j$|} t%| d}!|jdB|jdC|t&dD||!|d.r\rrTrrrrrFrrrNrrlr]rrRrrrtrrrrrBrrrrrrrrrr>rrrrrrrrrrrrrrrrrrranalysis_cost_estimation)r word_countdocument_countrgrrrr)rrrr)rrrr)'rrrlrrNrrrrrrrrrrrerr4rhrrr`r rrrrTrrrrrrrrrrrfr_)"rDrYrFrVrnrtrrrrrrrrrrkrrrrrrrrrrrrrrrrrrr:r:r;estimate_analysis_costs     *                         z)MinaionsRFPService.estimate_analysis_cost folder_listc sz1|djd|tjjd|d}tj|ddtj|d|}tj|dddiiij | d iirr D]\}}t |t red |vre||d <qRjd td f d d|D]}||qtj|d} z+t| ddd} tj| dddWdn1swYjd| Wnty} zjd| WYd} ~ nd} ~ wwr5tj|d} z6t| ddd} tj| dddWdn1swYjd| jdtWnty4} zjd| WYd} ~ nd} ~ wwt} | dkrgjd| dt||t}|dr[jd njd!|d"n jd#ddi}tj|d$}}tj|rzHt|d%dd } t| }Wdn 1swYr߈ D]5\}}||vr||}||d&<|||<jd'|d(|qjd)|d*|q| D]\}}d |vrMd} D]\}}||kr||d <d}nq|s(|vr(|} D]\}}||kr&||d <d}nq|sMtj|rC||d <jd+|n jd,|d-| d&|kr\||d&< |}|rv|||<jd.|d/|q|||<jd0|qz)t|ddd} tj|| dddWdn 1swYjd1Wnty} zjd2| WYd} ~ nd} ~ wwWnty} zjd3| WYd} ~ nd} ~ wwjd4j||d5d6d7td5d8id9||d:d;}z t |W|WSty2} zjd<| WYd} ~ |WSd} ~ wwtyH} z t!d=| d} ~ ww)>z4Analyze Company documents to extract key informationcompany_doc_analysis Company Id: company_rTrcompany_merged_r rawAnalysis file_hashrsz! existing file hashes in databasec stj||d}tj|dd|dgD]}|di}|d}|di}|d}|d }|r7|s8qtj||} j|d } | rQ| d ksQ| d krRqt| d  } | | Wdn1sgwY|rr|| < | } | vr| } j d| d| |r| vr|  } | |d<| |d<||<|| <j d|t| n"| vr׈| }|| <t| j d| d|n| | <d7q|dgD]}||qdS)N folder_nameTrr documentIdr]rrrFsFalseFalserz"Cross-request duplicate detected: z matches existing file_id: rrz3Copied existing analysis to new duplicate file_id: z0Current-request duplicate detected and removed: z -> original: r subfolders)rrrrrNrrrr_calculate_file_hashr4r`copy_transform_file_pathr)folder base_path folder_path file_entryrkfile_idstoragerr local_filerrrexisting_file_idduplicate_analysis original_file subfolder directory_map duplicate_mapexisting_analysisexisting_file_hashesr file_hash_mapprocess_folderrDrFr:r;rsR              z?MinaionsRFPService.analyze_company_docs..process_folderzdirectory_map.jsonwutf-8encodingrFindent ensure_asciiNzDirectory mapping saved to z"Failed to save directory mapping: zduplicate_map.jsonzDuplicate mapping saved to z4Total current-request duplicates found and skipped: z"Failed to save duplicate mapping: z Processing z unique filesrhz"Processing completed successfully!zProcessing failed: rhz6No unique files to process - all files were duplicateszraw_analysis.jsonrrzCopied analysis from z to duplicate zOriginal file z% not found in analysis for duplicate zCalculated missing hash for z"Could not determine file_hash for z* - file not found and no mapping availablezMapped z -> zNo file_id mapping found for: z/Updated raw_analysis.json with file_id mappingsz*Failed to save updated raw_analysis.json: z%Failed to process raw_analysis.json: zUpdating Company Info in DBprocessing_completed)rrirWrYrVrc processed)rVrFrriz&Failed to cleanup download directory: z%Failed to analyze Company documents: )"rr4r`rrrrrrrXrNr7rbrdrrrPdumprrgmdpprocess_directoryrrhrrrrrrrrrr_)rDrFrrVrrr r root_folderdirectory_map_filerrduplicate_map_fileunique_files_countresultsraw_analysis_filertemp_raw_analysisduplicate_path original_pathrr hash_foundhash_valr results_dictr:rr;analyze_company_docss O               z'MinaionsRFPService.analyze_company_docsc) Cs`zy|||d|jd||j|}|s!td|t|d|kr>|jd|dd|td|t|d}|jd||j||}|s_td |d|j |d d id t d did| dsdd| dgD}|j |}t j|jd|} t j| ddg} |D]>} z| dd| d} | di} | | dWqty}z|jd| dd|WYd}~qd}~wwtj| | |jd}t j| d}t j|ddt j|d }t j|d!}d"|d#}|j||d$}t|d%kr8|d&kr8t|d' }||Wdn 1s3wYd"|d(}|j||d$}t|d%krm|d&krmt|d' }||Wdn 1shwYz|jd)t| ||jd*d+|jd}|jd,|rt|nd%d-d.}t|d/ }|}Wdn 1swYd"|d(}d0}|j||||} d.}t|d/ }|}Wdn 1swYd"|d#}d1}|j||||} Wnty }z|jd2|d}WYd}~nd}~ww||| |std3|jd4t !||}tj"|||jd\}}|jd5t#j$||jd}|D]H}|d6}|d7}|d8} |d9}!t%t&| |!d:}"| |!krl|d;| d<|}#n |d=| d>|!d<|}#||t#j'j()|#||"d?qA|j*|d||t|t|t+d@dA|,DdBdCid t dDdidt|}$n| di dE}| di dF}| di dGi dH}$|-||}%|jdI|%|jdJt|%dK|jdLtj.||%|jd\}&}'|jdM|&|||&|'dNdO}(|j |dP|(dQd t dR|&|$dSdTidzt/0| WntyO}z|jdU|WYd}~nd}~ww|&rc|jdV| dWdX|'n|jdY| dWdX|'dP|&|'||dZWSty}z(t1d[|z|j |d d\id t d]d^t|idTidWYd}~ww)_zFAnalyze RFP documents to extract key information and check eligibilityrrAnalysis record not found: r~Tenant mismatch:  vs rn)✅ Analysis found and tenant validated: r\rianalysis_in_progressrWrfp_analysis_startedrYr#cSrr:r_r`r:r:r;rbrz2MinaionsRFPService.analyze_rfp..r\ analysis_Trrrlr]rrRrrNrtanalysis_outputrrrrFrrrrz9Attempting document extraction with external extractor...rrBzDocument extractor returned rkrrrr>zDocument extractor failed: z8No text was extracted from documents using either methodz-Starting tender document analysis with LLM...z"Tender document analysis completed annexure_name document_name page_startpage_endrz on page z of z on pages -)rrar-source_documentpagescss|] }t|VqdSr)rsplit)rar|r:r:r; Nz1MinaionsRFPService.analyze_rfp..)documentsProcessedanalysisChunkstotalWordCount)extracted_info required_docsanalysisMetadatatender_analysis_completedrJrKrLrGrrz characterszStarting eligibility check...zEligibility check completed: ) isEligiblereason) extractedInforequired_documents eligibilityanalysis_completed)rir,rfp_analysis_completed) is_eligibledocuments_processedrg'Failed to cleanup temporary directory: u$🚀 Company is eligible for tender r^z . Reason: u*⚠️ Company is NOT eligible for tender )rirUeligibility_reasonrJrQzFailed to analyze RFP: analysis_failedrfp_analysis_failedrh)2rr4r`rrrrfrhrlrrrNrrrrrrrerrrrrrrr rrrrg_debug_extraction_resultseaaextract_annexures_infoanalyze_tender_with_LLMbpaidentify_required_documentsrcrange DocumentTypeANNEXUREupperrrsumvaluesrTcheck_eligibilityrrr_))rDrrFrVrrYrnrtrrrrkrrrrrrrrrrrrrannexure_hintsrJ doc_analysesrKannexurer=r>r?r@rCr-rVrrUrXanalysis_resultr:r:r; analyze_rfpsn     *       "                       zMinaionsRFPService.analyze_rfprKcCs|||d|j|}|std|t|d|kr3|jd|dd|td||jd||ddkrGtd |d i}|d i}|d i}d }dd|D} dd| D} dd| D} t | } t | } t | }d}g}| D]}dt dd}||7}| |dddt |ddq| D]}dt dd}||7}| |dddt |ddqt ||d}|t|d|dd |d!|| | |t |d|| d"td#||t |d|d$d%gd& }|j|}t|d'}|jd(|jd)||jd*||jd+t |d|jd,|d-| d.| d/|jd(||d!|t |d|| | || d0d1S)2z-Estimate the cost for bid document generationestimate_bid_generationr5r~r6r7r8rirSz@Analysis must be completed before estimating bid generation costr,rRrPicSs$g|]}|dddvr|qS)rar)CUSTOMrcrNrdrr:r:r;rb$zCMinaionsRFPService.estimate_bid_generation_cost..cS$g|]}|dddkr|qS)rarrnrorr:r:r;rbrpcSrq)rarrcrorr:r:r;rbrpr}rrrAnnexurercr)documentracostdzCustom Documentrnrnbid_generation_estimationrr)totalBillableDocumentscustomDocumentCount annexureCountradditionalCost costBreakdownbillableDocumentsbid_generation_cost_estimation)rradditional_costbillable_document_countrg) r~rn analysisIdrarirrgenerationMetricsrWr]z2==================================================u"Estimated Bid Generation Cost: ₹uBase Cost: ₹uAdditional Cost: ₹zBillable Documents: z (Custom: z , Annexure: rp)rrtotal_billable_documentscustom_document_countannexure_countcost_breakdownbillable_documents)rrrr)rrrrrfr4rhr`rNrrandomuniformrerrr)rDrrFrKrr,rRrJbase_generation_cost billable_docs custom_docs annexure_docs custom_countrtotal_billable_countrrrj annexure_cost custom_doc custom_costrbid_generation_recordbid_generation bid_gen_idr:r:r;estimate_bid_generation_costs               z/MinaionsRFPService.estimate_bid_generation_costrc"Cs>|||d|jd||j||}|s td|t|d}|j|}|s5td|t|d}|j||} | sMtd|d|j |dd id t d d id zt j |jd|} t j |jd|} t j| ddt j| dddd| dgD} |j| } | D]{}zW|jt|d|}|dd|d}|ds|diddkr|d}t j | |}t|d }||Wdn1swY|jd|Wqty}z|jd|dd |WYd}~qd}~ww|||}||| ||}t j | d!}t j|ddt j |d"}t|d#d$d%}tj|d&id'i|d(d)Wdn 1s[wY|jd*| d+tj| | |||jd,\}}}||||| d+d-}||||| d+d-|}z t !| t !| Wnty}z|j"d.|WYd}~nd}~wwg}g}|D]&}|d/d-}|d0d-|d1d-|d2d-|d3d-|d4} |#| q|jd5|||}!|j ||!|d6d7||d8d t d9t$|!t$|t$||t$|d:d;id<|jd=t$|d>t$|d?d@|!||t$|!dAt$|d>t$|dBdCWSty}zLt%dD|z&dEt&vrit j '| rit !| dFt&vr{t j '| r{t !| WnY|j |ddGid t dHdIt|id;id d}~ww)Jz9Generate complete bid documents using enhanced automationrr!Bid generation record not found: rr5rnr\rigeneration_in_progressrWbid_document_generation_startedrYr#bid_gen_ company_docs_TrcSrr:r_r`r:r:r;rb_rz=MinaionsRFPService.generate_bid_documents..r\r]rrlrrmimeTyperrNrqFailed to save document rtender_analysistender_analysis.jsonrrrr,rPr z$Generating bid documents for tender r^rtrz)Failed to cleanup temporary directories: r>rrr{ra)rrr{rar>zcompany document urls )generatedDocumentscompanyDocumentUrlsgeneration_completed)rioutputDirectorydocumentStatus!bid_document_generation_completed)rgenerated_countreferenced_countoutput_directorycompany_docs_countrg)rrVrczSuccessfully generated z bid documents and referenced  company documents completedz Generated z company documents successfully)rigenerated_documentscompany_document_urlsrrrz"Failed to generate bid documents: temp_rfp_dirtemp_company_docs_dirgeneration_failedbid_document_generation_failedrh)(rr4r`rrrrfrrlrrrrrrrrNrrrrrrrhrT(_download_and_organize_company_documentsrPr%r_prepare_bid_documents_save_generated_documents_to_s3#_create_referenced_document_entriesrrrgrerr_localsr)"rDrrFrKrVbid_genrrrYrnrrrtrrkrrrrrformatted_company_infodownloaded_company_docs analysis_dir analysis_filefinal_docs_dirdocument_statusreferenced_company_docsgenerated_document_idsreferenced_document_idssimplified_company_docscompany_docs_urlsr>simplified_docall_document_idsr:r:r;generate_bid_documents6s         * "      "     z)MinaionsRFPService.generate_bid_documentsr user_promptc+Cs|||dz|j||}|std|t|d}|j|}|s.td|t|d} |j| |} | sFtd|d|j|g} | sVtd|| d} | d d |} |j d | d |d t j |jd|}t j|ddz|j||}t j || }t|d }||Wdn1swY|j d|dd| dgD}|j|}|D]x}zT|jt|d|}|d d |d}|ds|diddkr|d}t j ||}||krt|d }||Wdn 1swYWqty@}z|j d|dd|WYd}~qd}~wwt j |d}t j|ddt j |d}t|d d!d"}tj|d#id$i|d%d&Wdn 1sywY|||}t j |d'}t j|ddt j |d(}t j|dd|j d)|d*|d+}|j d,|tj|||||d#id$id-\}}|std.t j |s4z7|j d/||j d0|d1t |D]\}} }!|!D]}"t j ||"}|j d2|qqWnty,}#z|j d3|#WYd}#~#nd}#~#wwtd4||j d5|t|d6 }|}$Wdn 1sRwYt j |}| d7t| }%t d8d9|%}%d:|d;|%d<|}&|!|}'|"d=rd>}'|j#|$|&|'|}(||d?d(d@|(||%|| |t$$dAdB})|j%|)}t|d}*|j&|dC|*idDt$$dE|*|| |dFdGidH|j dI|dJ|*|| ||&dK|dLWzt'(|WWSty}z|j )dM|WYd}~WSd}~wwzt'(|Wwty&}z|j )dM|WYd}~wd}~wwtyY}z&t*dN|zdOt+vrLt j |rNt'(|WWWYd}~ww)Pz9Regenerate a specific bid document based on user feedbackregenerate_bid_documentrrr5rnr\Document not found: rrrlzRegenerating document: rorp regenerate_TrrNzDownloaded document locally: cSrr:r_r`r:r:r;rb. rz>MinaionsRFPService.regenerate_bid_document..r\r]rrrrz Failed to save context document rrrrrrr,rPrrregenerated_docs regeneratedzCreated output directories: z and gz+Calling BPA regenerate_document with path: )org_doc_file_pathrrrrJz7Failed to regenerate document - no output path returnedzRegenerated file not found at: z Contents of :z Found file: z"Error listing directory contents: z1Regenerated document not found at expected path: z&Successfully regenerated document at: rr^rurbid_documents/r=z /regenerated/r.htmrbid_document_regeneratedrx)bidGenerationIdr^originalDocumentIdoriginalDocumentName userPrompt regeneratedAtr}regeneratedDocumentsrWdocument_regenerated)roriginal_document_idoriginal_filenamerrgrz#Successfully regenerated document: rz#Document regenerated successfully: )riregenerated_document_idrrregenerated_filenamerrrWz#Failed to regenerate bid document: r),rrrrrfrrlrrNr4r`rrrrrrrrrrrhrPr%rTr_regenerate_documentrrrrrrrrrrrrrrrgr_r)+rDrrFrrrVrrrrYrnrrtrrrlocal_document_pathrrttender_documentsrkrrrrrrrregenerated_subdir regen_costregenerated_doc_pathrdirsrr debug_errorrrrrrrregenerated_doc_idr:r:r;rsF           *"           z*MinaionsRFPService.regenerate_bid_documentrcCsbtj|s td||j||}|dsdnd|d|d|d|dd|dd d S) z?Upload company documents to S3 for future use in bid generationz%Local documents path does not exist: upload_errorsrpartialuploaded_filesuploaded_count error_countz Uploaded z company documents to S3)rirrrrr)rrrrrr)rDrFrresultr:r:r;upload_company_documents_to_s3 s z1MinaionsRFPService.upload_company_documents_to_s3c Cszo|||d|j|}|std|t|d|kr'td|t|d}|j||}|s?td|dztj|j d|}tj |ddd d | d gD}|j |}|D]C} |j t| d |} | d d| d } | ds| d} tj|| } t| d } | | Wdn1swYqf|jd|tj|dd}td|tj|st|}nd}|rVd|}d}tj|s||tj|s|||||rRt|||}d}t|d } | }Wdn 1swYd|d}d}|j||||}|j|d|d|d|did td!d"|id#id$d%||d&d'WWSt d(t d)t yp}z |j!d*|d}~wwt y}z t!d*|d}~ww)+z>Setup chat functionality for RFP queries using enhanced searchsetup_rfp_chatr5r~rnr\ chat_setup_TrcSrr:r_r`r:r:r;rb rz5MinaionsRFPService.setup_rfp_chat..r\r]rrlrrNz+Setting up chat functionality for analysis r chunks.jsonChunk Data file path r; index_es.jsonrrr /chunks.jsonr> chatSetuprR)bidIdsetupCompleted chunksPath indexConfigrWchat_setup_completedbid_idrgr#rz%Chat functionality setup successfully)rir chunks_pathrFailed to validate chunks data*Failed to process documents for chat setupzFailed to setup RFP chat: )"rrrrrfrlrrrrrrNrrrrrr4r`r_rbq process_task_create_default_index_config_convert_chunks_to_json_validate_chunks_datachat es_index_datarrrrrrh)rDrrFrrYrnrrtrrkrrrr data_fileririndex_config_file tag_responsesrrrrrr:r:r;r s                    z!MinaionsRFPService.setup_rfp_chatcCs|j|}|std|t|d}|j||}|s'td|dd|d}|j||d}t|tr?|d}d|}| d rQ|d  d st |d krt d |j|d |d dddtdidtddid|j|ddtd|dd ddiidSzt d||||dtj|jd|} tj| d ddd| dgD} |j| } |j d t| d!| D]D} |jt| d"|} | d#d$| d"}|d%s|d%}tj| |}t|d& }|| Wdn 1swYq|j d'tj| d(d)}t d*|tj|s:t | }nd }|r5d+}tj|sM| |tj|sZ|!| ||j d,|"|r1ztt#$|||}d}t|d- }|%}Wdn 1swYd|d.}d/}|j&||||}|j|d |d d|d0|tdidtddid|j|ddtd|t|d ddii|j d1|Wn^t'y0}zI|j(d2||j|d |ddd t|td3idtd4t|d5d6did|j|ddtd4t|d5d6diiWYd}~n d}~wwt'd7t'd8z t)*| WWdSt'y^}z|j+d9|WYd}~nd}~wwWdSt'y}ztt (d:|d;|z?|j|d d|ddd t|td3idtd4t|d|j|ddtd|d d?dii|j d1|dS)@z1Background task for setting up chat functionalityr5rnr\rrFrr;rrpzRFP documents small size...Tr)rrsetupInProgressrrsetupCompletedAtrWrrYr#rcr)rindexed_chunksrhrgNz,Starting background chat setup for analysis setup_rfp_chat_bgrrcSrr:r_r`r:r:r;rb rz@MinaionsRFPService.setup_rfp_chat_background..r\z Downloading z documents...r]rrlrrz&Processing documents for chat setup...rrrrz"Starting Elasticsearch indexing...rrr>rRz/Chat setup completed successfully for analysis zElasticsearch indexing failed: )rrr  setupFailed errorMessagefailedAtchat_setup_failedelasticsearch_indexing)rhstagerr"Failed to cleanup temp directory: z*Background chat setup failed for analysis rbackground_processingrz/Chat setup already completed. Reusing the same.)rrh),rrrrfrlrrbrdecoderNrrr_r`rrrrrrrrrrrr4rrrrrrrrrrrrrrrrhrrrg)rDrrFrrYrnrmerged_txt_filerrrtrrkrrrrrrirrrrres_error cleanup_errorrrr:r:r;setup_rfp_chat_background\ s                        *     , z,MinaionsRFPService.setup_rfp_chat_backgroundquery client_idc Cs|j|}|std|t|d|krtd|d|d}|j||d}t|tr6|d}t |dkr@d} nd} |j ||} | sRtd || d i} | d s|| d i} | sitd |j |d | id t ddid| d} |sd|}zntj|jd|} tj| dd}d| d}tj|stj| d}tj|dd|j||d}t|d }||Wdn1swYt|| ||| |}|j|ddt |||dii|WSty }z td|d}~ww)z8Chat with RFP documents using enhanced search and Clauder5r~rrFrr Tr\rrz6Chat functionality not setup. Please setup chat first.rWrrYr#rtenant_rrrrrrNrc chatHistory)rZclientIdrrSzFailed to process chat query: )rrrrfrrbrrrrrlrNrrrrrrrrrrrr chat_with_rfprrr_rh)rDrrFrYrrrrr no_es_queryrn chat_setuprrrrrrrSrr:r:r;r z sx              z MinaionsRFPService.chat_with_rfpTrdocument_categoriesadd_page_numbersc%Cszs|||d|std|j||}|std||}|j|}|} | s/td|jdt| dtj |j d|} tj | d} tj | d d tj | d d tj | d } tj | d d zg} | D]}zk|j t|d |}|d d|d }tj | |}t|d }||Wdn1swY|dr||||| }|rtj |r| ||jd|n |jd|Wqoty}z|jd|d d|WYd}~qod}~ww| stdt| D]!\}}tj |}|dd|}tj | |}t||q t|dd}|j||}|rC|ddnd}t !dd|}t"t##d}|d|d }tj | |}|jd!t$%| ||tj |s{td"t|d# }|&} Wdn 1swYd$|d%|d&|}!|j'| |!d'|}"||d(d)d*|"||t| t| ||pd+t##|d,d-}#|j(|#}t|d }$|j)|d.|$id/t##d0|$t| t| ||d1d2id3|jd4t| d5t| d6|d7|$|t| t| t| |"d8d4t| d9t| d:d;Wz t*| t*| WWStyK}z|jd<|WYd}~WSd}~wwz t*| t*| Wwtyt}z|jd<|WYd}~wd}~wwty}z |jd=|d}~ww)>a Merge bid documents into a single PDF with index Args: bid_generation_id: Bid generation record ID tenant_id: Tenant ID generated_document_ids: List of generated document IDs to merge document_categories: List of document categories to include (optional) add_page_numbers: Whether to add page numbers to merged PDF Returns: Dict containing merged document information merge_bid_documentsz"No generated document IDs providedrzNo documents found to mergeMerging z) documents (will convert non-PDFs to PDF)merge_ convertedTrr$r]rrlrNrzSuccessfully processed: zFailed to convert document: zFailed to process document rz/No documents were successfully converted to PDFrvrrnrr^bidrui_merged_documents_rzStarting PDF merge process...*PDF merge failed - output file not createdrrr=/merged/rbid_document_mergedmergedrxall)rr^mergedDocumentCountconvertedDocumentCountaddedPageNumbersmergedCategoriesmergedAtprovidedDocumentIdsr}mergedDocumentsrWdocuments_merged)merged_document_idrconverted_countoutput_filenameprovided_document_idsrgrSuccessfully merged z documents (z converted to PDF) into rrRz documents (converted z to PDF))rir8rrr9 file_sizes3_urlrrzFailed to merge bid documents: )+rrrrrr4r`rrrrrrrrfrNrrrr_clean_html_file_convert_to_pdfrrergrrhr5rrrrlrrrr pdf_mergermerge_pdfs_with_indexrrrrr)%rDrrFrr#r$rgenerated_doc_idsrprocessable_documentsrconversion_dir processed_dirconverted_pdf_filesrkrroriginal_file_pathrconverted_pdf_pathrindexpdf_filer new_filename dest_pathrYrnrrZr: output_pathmerged_contentrrr merged_doc_idr:r:r;r%~ s      *      &   z&MinaionsRFPService.merge_bid_documents2target_size_percentpreserve_qualitycCsz|||d|dvrtd|j|g}|s!td||d}|dd|}|ds9td |jd |d |d t j |j d |} t j | ddz<|j||} t| } t j | |} t| d } | | Wdn1swYt j |d}|d|d}t j | |}|jdtj| ||d|d\}}}|std|ddt|d } | }Wdn1swYt|}| || d}|di}|dd}tdd |}d!|d"|d#|}|j||d$|}||d%d&d'|i||| |||||td(d)}|j|}t|d*}|j|d+|id,td-|d.||d/d0id1|jd2|d3d4d5||||| |||d6d|d7|d8|d9|d3d d: Wzt| WWSty}z|j d;|WYd}~WSd}~wwzt| Wwty}z|j d;|WYd}~wd}~wwty}z |j!d<|d}~ww)=a Compress a PDF document Args: document_id: Document ID to compress tenant_id: Tenant ID target_size_percent: Target size as percentage of original (25, 50, or 75) preserve_quality: Whether to prioritize quality over exact size target Returns: Dict containing compressed document information compress_documentrQK$Target size must be 25%, 50%, or 75%rrrrlrz$Only PDF documents can be compressedzCompressing document: z to % compress_TrrN _compressed_ percent.pdfzStarting PDF compression...r) max_attemptsrSzPDF compression failed: rhz Unknown errorrrvrr^rtrurrr= /compressed/rbid_document_compressed compressedrx)r originalSizecompressedSizecompressionRatiotargetSizePercentpreserveQualitycompressionStats compressedAtr}r]compressedDocumentsrWdocument_compressedr)compressed_document_idrr:provided_document_idrgrz"Successfully compressed document: .1f % reductionrachieved_percentrR Compressed z by ) rirrjrcompressed_filename original_sizecompressed_sizecompression_ratiotarget_achievedr>compression_statsrrzFailed to compress document: )"rrrrrNrrr4r`rrrrrrrrrrrA compress_pdfrrrrrrrrfrrrrgrh)rDrrrFrRrSrrtrrrrq input_pathr base_namer:rNrh final_pathstatscompressed_contentrrrsrrrrrrkcompressed_doc_idrr:r:r;rTs            z$MinaionsRFPService.compress_documentc&sbz|||d|dvrtd|j||}|s!td||dg}|s-td|j|}r>fdd|D}d d|D} | sKtd |jd t| d |d t j |j d|} t j | ddzg} i} | D]~} z[|jt| d|}| dd| dd}|ds|d7}t j | |}t|d }||Wdn1swY| |t| d| |<|jd|Wqtty}z|jd| dd|WYd}~qtd}~ww| std|jdt| |dg}d}d}t | D]}|d|dr|d|dd}| |}|s1qt j | |}t j | |}t j |rt j |rt|d  }|}Wdn 1sbwYt j |}t|}|||d!}||7}||7}t|d"d#}|j||}|r|d$d%nd%} t d&d'| } d(|d)| d*|}!|j!||!d+|}"||d,d-d.|"|| |||||||t""d/ d0}#|j#|#} t| d}$||$|||||||"d1d2|jd3|d|d4d5q|dkr|||d!nd}%|j$|d6d7t""d8t|||||%d9d:ii|jd;t|d<|jd=|%d4d5d>|t||||%|d?t|d@|%d4dAdBWzt%&| WWStyu}z|j'dC|WYd}~WSd}~wwzt%&| Wwty}z|j'dC|WYd}~wd}~wwty}z |jdD|d}~ww)Ea Batch compress all PDF documents in a bid generation Args: bid_generation_id: Bid generation record ID tenant_id: Tenant ID target_size_percent: Target size as percentage of original (25, 50, or 75) preserve_quality: Whether to prioritize quality over exact size target document_categories: List of document categories to compress (optional) Returns: Dict containing batch compression results batch_compress_documentsrUrXrrz(No generated documents found to compresscsg|] }|dvr|qS)r)rNrr#r:r;rbszCMinaionsRFPService.batch_compress_bid_documents..cS&g|]}|dddr|qSrrrrNrrrr:r:r;rb&z"No PDF documents found to compresszBatch compressing z PDF documents to rYbatch_compress_Trr]rrlrrNzDownloaded for compression: Failed to download document r)No documents were successfully downloadedz!Starting batch PDF compression... _compressedrr[r\rrvrnrr^r)rurrr=r^rr_r`rx) rr^roriginalFilenamerarbrcrdrergr}rR)rjrrrprqrrrsr>rorlrmrcrWbatch_documents_compressed)compressed_countrRtotal_original_sizetotal_compressed_sizeoverall_compression_ratiorgzSuccessfully batch compressed rkzOverall compression: rzSuccessfully compressed z documents with z% overall reduction)ricompressed_documentstotal_documentsrrrrRrrz$Failed to batch compress documents: )(rrrrrNrr4r`rrrrrrrrfrrrrrerrhrAbatch_compress_pdfslistdirrrrgetsizerlrrrrrrrrrg)&rDrrFrRrSr#rrCr pdf_documentsrrdoc_id_mappingrkrrrrrcompressed_docsrrroriginal_doc_idcompressed_pathr0r{rqrrrsrYrnrrrrr|overall_compressionr:r~r;batch_compress_bid_documentss$     *          z/MinaionsRFPService.batch_compress_bid_documentsinclude_linked_docscCsz|||d|j||}|std|dd|dgD}|s*td|j|}|r:dd|D}ndd|D}|sGtd |jd t|d t j |j d |} t j | d dzgg} |D]v} zS|jt| d|} | dd| dd} | ds| d7} t j | | }t|d }|| Wdn1swY| ||jd| Wqkty}z|jd| dd|WYd}~qkd}~ww| std|dd}tdd|}|d}t j | |}|jdt| ||t j |stdt|d  }|}Wdn 1s.wYd!|d"|d#|}|j||d$|}||d%d&d'|||t|||td(d)}|j|} t| d}|j |d*|id+td,|t||d-d.id/|jd0t|d1|d2||t|t||d3|d0t|d4d5Wzt!"| WWSty}z|j#d6|WYd}~WSd}~wwzt!"| Wwty}z|j#d6|WYd}~wd}~wwty}z |jd7|d}~ww)8a Merge all tender/RFP documents into a single PDF with index Args: tender_id: Tender ID tenant_id: Tenant ID add_page_numbers: Whether to add page numbers to merged PDF include_linked_docs: Whether to include linked documents Returns: Dict containing merged document information merge_tender_documentsr\cSrr:r_r`r:r:r;rbrz=MinaionsRFPService.merge_tender_documents..r\zNo documents found in tendercSrrrrr:r:r;rbrcSsBg|]}|dddr|ddkr|ddkr|qS)rrrrrnrarwrrr:r:r;rbszNo PDF documents found to merger&z tender PDF documents merge_tender_Trr]rrlrrNz Downloaded: rrrr^rnrurz_merged_tender_documents.pdfz$Starting tender PDF merge process...r+rrvr=r,r tender_mergedr.rx)rzr^r0r2includeLinkedDocsr4r}r6rWtender_documents_merged)r8rr:rgrr<z tender documents into rrRz tender documents)rir8rrr=r>rrrz"Failed to merge tender documents: )$rrrlrrNrr4r`rrrrrrrrfrrrrrerrhrrrArBrrrrrrrrrrg)rDrYrFr$rrnrtrrrrrkrrrrrrr:rNrOrrrrPr:r:r;ros    *         z)MinaionsRFPService.merge_tender_documentsrrc Cstdtd|tdtj|rt|nd|rZtdt|d|D]'\}}|rA|ddd d nd }td tj |d t|d|dq1nt dtddS)z&Debug helper to log extraction resultsz&=== Document Extraction Debug Info ===zTemp directory: zFiles in temp directory: zDirectory not foundzExtracted text from z documents:Nrxr rz No contentr3rz chars - ...z*No documents_text returned from extractionz=== End Debug Info ===) r_r`rrrrrr7rrrg)rDrrdoc_pathrcontent_previewr:r:r;r[s &, z,MinaionsRFPService._debug_extraction_resultsrc Cs4tj|d}tj|rzot|ddd }t|}Wdn1s%wYtdt |dt |t ru|rui}| D]\}}t |t rT|rT|||<qBtd|qB|rmtd t |d |WStd WiStd WiSty}ztd |WYd}~iSd}~wwiS)z5Check and load results from document extractor outputrrrrNz"Found existing doc_text.json with z entrieszSkipping invalid entry for zLoaded z) valid documents from existing extractionz0No valid content found in existing doc_text.jsonz.doc_text.json exists but contains invalid dataz&Error reading existing doc_text.json: )rrrrrrPrr_r`rrbrdr7rfstriprgrrh) rDr doc_text_filerrK valid_entriesrrrr:r:r; _check_document_extractor_output%s4     z3MinaionsRFPService._check_document_extractor_outputrc Cs8tj|d}ddddddddd d }||d S) rrrrrrzapplication/mswordrzapplication/vnd.ms-excelr>) rrrrr.docr.xlsrrrrr:r:r;rGs z$MinaionsRFPService._get_content_typerrc CsPg}t|D]\}}}|D]} tj|| } | ds!| dr"qzet| d } | } Wdn1s7wYt dd|}tj | |} d|d|d| }| | }|j | |||}|| d tj|d |||| d d }|j |}t|d }||Wqty}ztd| d|WYd}~qd}~wwq|S)z>Save generated bid documents to S3 and create database recordsrrrNrurrr=rmrx)rr^ relativePathr}r]z"Failed to save generated document r)rrrrrrrrrrrelpathrrrrrrfrerr_rh)rDrrrFrrtrrrrrrrrel_pathrrrrtrkr~rr:r:r;rWsJ      ",z2MinaionsRFPService._save_generated_documents_to_s3rrcCsg}i}|D]'}|d} | r-|d|d|d|dd|d|dd || <q|D]} z| d} || } | sM|jd | d Wq0| d} d d l} | j| }|jd}d|jvrg t dd|}|| |d}| dd ur| d|d<| dr| d|d<|| d | dd| d| dd|||dd| dd| d| dddd}|j |}t |d}|||jd| d d|dWq0ty}ztd| d d|WYd }~q0d }~ww|S)a  Create database entries for referenced company documents without uploading to S3. Uses the original S3 URL from the company documents. Args: referenced_company_docs: List of referenced company documents from bid preparation bid_generation_id: Bid generation ID tenant_id: Tenant ID bid_number: Bid number downloaded_company_docs: List of downloaded company documents with S3 URLs Returns: List[str]: List of created document IDs local_pathr>rrr-rrr)r>rrr-rrz/Could not find S3 URL for referenced document: rrNr=z.s3.rur)rrRrrmrxTrequired_doc_name)rr^isReferencedDocumentoriginalCompanyDocrequiredDocNamer-r}r]z#Created referenced document entry: rorpz/Failed to create referenced document entry for r)rNr4rg urllib.parseparseurlparserlstriphostnamerrrrrfrer`rr_rh)rDrrrFrrrtlocal_path_to_s3 company_docrref_docs3_infor>urllib parsed_urlrbid_number_saferrtrkr~rr:r:r;rst                $(z6MinaionsRFPService._create_referenced_document_entriescCsddddddiddidd d dd id d d}tjtj|ddt|d}tj||ddWddS1s=wYdS)z0Create default Elasticsearch index configurationrr)number_of_shardsnumber_of_replicastrueenabledrar| dense_vectori)radimskeyword)r| title_vectortag)dynamic_source properties)settingsmappingsTrrrN)rrrdirnamerrPr%)rDrrrr:r:r;rs( "z/MinaionsRFPService._create_default_index_config output_filec Csddl}tj|dd}tj|rZ||}g}|D]\}}||dd|dd|ddd qt |d d d } t j || d ddWddS1sSwYdSdS)z>Convert Excel chunks to JSON format for Elasticsearch indexingrNrz chunks.xlsxTagrquestionanswertagNamerrrrrrFr) pandasrrrr read_exceliterrowsrerNrrPr%) rDrrpd excel_filedf json_datarrowrr:r:r;rs      "z*MinaionsRFPService._convert_chunks_to_json process_id process_typecCsd||_||_||_td|d|||||_t|dr$|jr$|j|j_|jd|d|dS)z Set up process context for logging Args: tenant_id: Tenant ID process_id: Process ID (tender_id, analysis_id, etc.) process_type: Process type (discovery, analysis, bid_generation) z rfp_service.rrz Starting z process for N)rrrr r4hasattrrr`)rDrFrrr:r:r;r*s  z)MinaionsRFPService._setup_process_contextdoc_infocst|dd}|dd|ddgd}||vs6tfdd|Ds6tfdd|Dr8d Sd S) a Categorize company document as standard or experience based on type and content Args: doc_info: Document information dictionary Returns: str: Category ("standard" or "experience") rarrr)r)rr' work_order completion portfoliorr&c3|]}|vVqdSrr:ra indicatorrr:r;rE[zBMinaionsRFPService._categorize_company_document..c3rrr:r) doc_valuer:r;rE\rrr)rNrany)rDrr/experience_indicatorsr:)rrr;_categorize_company_documentEs z/MinaionsRFPService._categorize_company_document target_dirc Cs(z|d}|s WdS|dd}tdd|dd}d dl}|j|}tj |j d p2d }||} tj || } | d } t j|d d} | t| d} | | jWdn1sewYd}|dr||dr||d}nLg}|dr|d|d|dr|d|d|dr|d|ddD]}||vr|dtj||ddqd |}t| ddd } | |Wdn1swY|jd| | WSty}z|jd|dd d!|WYd}~dSd}~ww)"a$ Download company document and create .desc file Args: doc_info: Document information dictionary target_dir: Target directory to save the document Returns: str: Path to saved document or None if failed r>Nrrtz[^\w\s-]rrrrrr.desc)timeoutrr-zName: razType: rzValue: ) director_dataexperience_dataitr_data ca_cert_databalance_sheet_datacertificate_datazData: rrr rrrzDownloaded company document: z$Failed to download company document unknownr)rNrrrrrrrrrrrrMraise_for_statusrrrrerPr8r4r`rrh)rDrrr>r safe_filenamerrfile_extensionrr desc_pathrSrdescription_content desc_partsrrr:r:r;#_download_and_save_company_documentbsT            z6MinaionsRFPService._download_and_save_company_documentc Cstd| zt|ddd }t|}Wdn1s!wYg}d}t|D]r\}}t|tsDtd|d|d7}q.d |vsL|d sYtd|d |d7}q.d |vsa|d sntd|d |d7}q.d |vsv|d std|d|d7}q.t |d  t |d  t |d  d}| |q.tdt |d|d|rt|ddd}tj ||dddWdn1swYtdt |dt |dkWSty} ztd| WYd} ~ dSd} ~ ww)NzValidated Data file path rrrrzChunk z: Not a dictionaryrrz: Missing or empty tagNamerz: Missing or empty questionrz: Missing or empty answerrzValidation complete: z valid chunks, z invalid chunksrrFrzWrote z cleaned chunks back to filezError validating chunks data: )r_r`rrPrr5rbrdprintrfrrerr%r) rDrrrK valid_chunks invalid_countrAchunk cleaned_chunkrr:r:r;rsT   z(MinaionsRFPService._validate_chunks_datarRcs\sdSdrtfdddDsdSddkrdSgd}tfd d|DS) z Check if URL is a valid file URL (not just a website) Args: url: URL to check Returns: bool: True if it's a file URL, False if it's a website F)z http://www.z https://www.c3|] }|vVqdSrr)rarrRr:r;rErFz8MinaionsRFPService._is_valid_file_url..)rrr.jpg.pngr=) documents/zfiles/zuploads/z attachments/rrrrrrrz.zipc3rrrrrr:r;rErF)rrcount)rDrRfile_indicatorsr:rr;_is_valid_file_urls  z%MinaionsRFPService._is_valid_file_urlrrc6 CsBz|j||}|di}|di}tj|d}tj|d} tj|ddtj| ddg} |r|jdt } | D]\} } |jd| | d }|r[| |s\qB|| vraqB| || d }| d d \}}td d|d}td d|d}|dr| }ddi}n|}ddi}tj| didd}| ddp|d}|||||d|d}||||}|r| |qBn|jd|dg}t|D]Z\}}|d}|r| |sq|dd|d }|dd}|d d}t|i}||||dd|d|d}|dd}|dkr)| n|}||||}|r:| |q|d!g} t| D]X\}}!|!d}|rW| |sYqEtd"i}|!dd#|d }"td$d|"d%d}#d#|#d&d"d'|!dd(d|d"||!d)}||||}|r| |qE|d*g}$t|$D][\}}%|%d}|r| |sqtd+i}|%d,d-|d }&td$d|&d%dd.d/}'d0|'d+d1|%d,dd|d+||%d2}||| |}|r| |q|d3i}(|(d4g})t|)D]F\}}*|*d}|r&| |s(qtd5i}|*d6d7|d }+d8|+d5d9|+d|d5||*d:}||||}|rY| |q|(d;g},t|,D]>\}}-|-d}|rw| |syqetddd|d<||-d?}||||}|r| |qe|(d@g}.t|.D]F\}}/|/d}|r| |sqtdAi}|/dBd7|d }0dC|0dAdD|0d|dA||/dE}||||}|r| |q|dFg}1t|1D]T\}}2|2d}|r| |sqtdFi}|2ddG|d }3td$d|3d%dd.d/}4dG|4dF|3d|dF||2dH}||||}|rR| |q|jdIt| dJ|jdKtdLdM| D|jdNtdOdM| D| WSty}5z|jdP|5gWYd.}5~5Sd.}5~5ww)Qa Download and organize company documents using centralized mapping (FIXED VERSION) Args: tenant_id: Tenant ID temp_company_docs_dir: Temporary directory for company documents formatted_company_info: Formatted company information for fallback descriptions Returns: List[Dict]: List of downloaded company documents with metadata rrStandard_DocumentsExperience_DocumentsTrz)Processing documents from raw_analysis...zRaw Analysis Item...rr/rrz [^a-z0-9]+rrrrr0F)r!r-rz document from raw_analysis)rrarr-r>r{mapping_configz@Starting company documents download using centralized mapping...rrr detail_doc_ra basic_detailsrrr( director_ [^\w\s\-_]r _documentz Director: Unknown)rrarr-r>r{r rr%r)r'project_NrQ experience_z Project: )rrarr-r>r{r rrrr%ryear_ITR_z ITR Year: )rrarr-r>r{r rr r&CA_Certificate_r!)rrarr-r>r{r rr"r'r#Balance_Sheet_r$)rrarr-r>r{r rr* Certificate_)rrarr-r>r{r rz&Successfully downloaded and organized rzStandard documents: cSg|] }|ddkr|qS)rrr:radr:r:r;rbzOMinaionsRFPService._download_and_organize_company_documents..zExperience documents: cSr)rrr:rr:r:r;rbrz3Failed to download and organize company documents: )rrWrNrrrrr4r`r6r7r r9rDrrrrrrPr8/_download_and_save_company_document_improved_v2rer5COMPANY_DOCUMENT_MAPPINGrrrrh)6rDrFrrrVr:rraw_analysis_docs std_docs_dir exp_docs_dirr seen_filesrrar raw_doc_type type_part name_partr/rrr  key_info_strr-r saved_infor<rAr=r>rrr(rB director_namesafe_director_namer)r project_namesafe_project_namerCrFrGitr_yearrHrIrJrKbs_yearrLrM cert_descsafe_cert_namerr:r:r;rs           .                 "                       "    z;MinaionsRFPService._download_and_organize_company_documentsr> default_namec CszCddl}|j|}|j}tj|}|rAt|dkrA|dsAtj|\}}t|dkr>| dd dd r>|WS|WS|WSt yN|YSw) a Extract meaningful filename from URL, fallback to default name Args: file_url: URL of the file default_name: Default name to use if extraction fails Returns: str: Meaningful filename rNrrrArr) rrrrrrrrrrisalnumr) rDr>r2rrrrr'rr:r:r;_extract_filename_from_url$s   $ z-MinaionsRFPService._extract_filename_from_urlc Csz,|d}|s|jd|ddWdS|dd}|||}||kr9t|dkr9tdd |}n(td d |d d }d dl }|j |} t j | j d} | s]d} || 7}t j ||} | d} |jd|tj|ddd} | d }| jdd}d|vr|dd }t| d}| jddD] }|||t|7}qWdn1swYd| jvrz t| jd}Wn ttfyYnw||}t| ddd }||Wdn1swY|di}|d d!}|jd"|d#|d$|d%|| |||||d&d|d'd||d( WStjjyS}z|jd)|ddd*|WYd}~dSd}~wtyv}z|jd+|ddd,|WYd}~dSd}~ww)-a IMPROVED VERSION - Download and save company document with concise, relevant descriptions Args: doc_info: Document information dictionary with mapping_config target_dir: Target directory to save the document formatted_company_info: Formatted company information for fallback descriptions Returns: Dict: Simple document information with local path or None if failed r>z No file URL found for document: rrNrtrQz [^\w\s\-_\.]rrrrrrrrzDownloading document: <T)rstreamz content-typer;ri ) chunk_sizezcontent-lengthrrrr rrz*Successfully downloaded company document: z (z ), size: z bytes, mimeType: r{ra) rr>rrr-r{rarrrz: Network error - rr) rNr4rgr5rrrrrrrrrrrrr`rMrrCrDr iter_contentrrr TypeError(_create_concise_description_from_mapping exceptionsRequestExceptionrhr)rDrrrr>r url_filenamerrrrrrrSr= mime_typerrrr rrr:r:r;rFs~           $    zBMinaionsRFPService._download_and_save_company_document_improved_v2c Cs|di}|dd}|rz|dd|dd|ddd}|d r@|d }||dd|d d|d dd nr|d r_|d }||dd|dd|dddnS|drt|d}|d|ddin>|dr|d}|d|ddin)|dr|d} |d| ddin|dr|d} |d| ddi|jdi|} | WSty} z|jd| WYd} ~ nd} ~ ww|dd} |dd}|r|S| r| Sd|ddS) a Create CONCISE description using mapping template - NO long company info repetition Args: doc_info: Document information dictionary Returns: str: Concise description using template r rrrr2rra)rrrarrr)rrrrr'r&r))r'r&rryearrrr#rr-r!rzTemplate formatting failed: Nz Document: rr:)rNupdaterrr4rgr)rDrr r template_varsrBrrGbsrIrMformatted_descriptionrr existing_descr:r:r;r<sn                     z;MinaionsRFPService._create_concise_description_from_mappingcs|dd|dd}|dd}tp!t|}|r*|ddSgd}|dvsA||vsAtfd d |DrCd SdS) a Categorize company document using centralized mapping (backward compatibility) Args: doc_info: Document information dictionary Returns: str: Category ("standard" or "experience") rrrar{rr) r)rr'rrrrr&contract)r)c3rrr:rrr:r;rErzKMinaionsRFPService._categorize_company_document_improved..r)rNrrr)rDrr/ doc_sourcer rr:rr;%_categorize_company_document_improveds  z8MinaionsRFPService._categorize_company_document_improvedrc$Cs$zptj|}tj|d}tj||d}tj|d}|dkr1t|||WS|dvrkz!ddl}dddddd}|j |||d |j d ||WWSt yj|j d |YWdSw|d vrRzddl} dd lm} m} m} m} m}ddlm}m}ddlm}ddlm}| |}| ||d}g}|}|jD]t}| j||d}|| d||d||dd|j !g|"d#t$j%!}t&|ddkrdd|D}| |}|'| ddd|j(fddd|j)fddd d!d"dd#d$|j*fd%dd$d|j+fg ||||dd&q|,||j d'||WWSt-yQ}z|j d(|d)|WYd}~WdSd}~ww|d*vrz]dd+l.m/}dd,lm} m} m}dd-lm}ddlm}||}| ||d}g}|}|j0D]}|j12r|| |j1|d.||ddq|,||j d/||WWSt-y}z|j d0|d)|WYd}~WdSd}~ww|d1vrfzidd2lm} m} dd-lm}ddlm}t3|d3d4d5 } | 4}!Wdn 1swY| ||d}|}|!5d6}"g}|"D]}#|#2r1|| |#2|d.q|,||j d7||WWSt-ye}z|j d8|d)|WYd}~WdSd}~ww|j 6d9|WdSt-y}z|j d:|d)|WYd}~dSd}~ww);a( Convert various file formats to PDF Args: file_path: Path to the input file output_dir: Directory to save the converted PDF Returns: str: Path to the converted PDF file rrrrNLetter2cm)z page-sizez margin-topz margin-rightz margin-bottomz margin-left)optionsz$Converted HTML to PDF using pdfkit: z&Pdfkit available for HTML conversion: )rr)SimpleDocTemplateTable TableStyle ParagraphSpacer)letterA4)colors)getSampleStyleSheet)pagesize) sheet_namezSheet: Heading1 rcSs.g|]}t|dkr|dddgn|qS)rZNr)r)rarr:r:r;rbgs.z6MinaionsRFPService._convert_to_pdf.. BACKGROUNDrrr TEXTCOLOR)ALIGNr\r^r^CENTER)FONTNAMEr\r]zHelvetica-Bold)FONTSIZEr\r])rdrrrarZ) BOTTOMPADDINGr\r]rYrfraGRIDr3zConverted Excel to PDF: zFailed to convert Excel to PDF r)rr)Document)rMrPrQ)rRNormalzConverted Word to PDF: zFailed to convert Word to PDF )r)rMrPrrrz zConverted text to PDF: zFailed to convert text to PDF z(Unsupported file format for conversion: zFailed to convert file to PDF )7rrrrrrrcopy2pdfkit from_filer4r` ImportErrorrhrreportlab.platypusrMrNrOrPrQreportlab.lib.pagesizesrRrS reportlab.librTreportlab.lib.stylesrU ExcelFile sheet_namesrrecolumnstolistfillnaastyperfrfrsetStylegrey whitesmokebeigeblackbuildrdocxri paragraphsr|rrrrDrg)$rDrrrname_without_extrNrrlmargin_optionsrrMrNrOrPrQrRrSrTrUrrkelementsstylesrWrrKtablerridoc_worddoc_pdf paragraphr text_contentrparar:r:r;r@s         "                     z"MinaionsRFPService._convert_to_pdfc CszZt|ddd }|}Wdn1swYtjdd|tjd}tdd|}t|d dd}||Wdn1sFwY|jd t j |WdSt yy}z|j d |d |WYd}~dSd}~ww) z Clean HTML file by removing markdown code block syntax (```html and ```) Args: file_path: Path to the HTML file to clean rrrNz```htmlr)flagsz```rzCleaned HTML file: zFailed to clean HTML file r)rrrr IGNORECASErrr4r`rrrrrg)rDrrrrr:r:r;r?s  &z#MinaionsRFPService._clean_html_filec Cs|d}dd|D}|std|ddd}|dd \}}td |td |||d}d||d d }d |d|d|} dtdtd| } | S)zv Transform local file_path into S3 public URL. Handles dynamic sub-paths after company_/. r=cSsg|] }|dr|qS)r)r)rapr:r:r;rbsz;MinaionsRFPService._transform_file_path..zCompany ID not found in pathrrrrrz Company ID:z Tenant ID:Nrz/company_docs/zhttps://rz.linodeobjects.com/)rDrrrrJrrr) rDrparts company_partrrVrF company_index dynamic_pathrr>r:r:r;rs    z'MinaionsRFPService._transform_file_pathcsbt}t|dtfdddD]}||qWd|S1s(wY|S)z Calculate SHA-256 hash of file content Args: file_path (Path): Path to the file Returns: str: Hexadecimal hash of the file content rcs dS)Ni)rr:rr:r;s z9MinaionsRFPService._calculate_file_hash..N)hashlibsha256riterrB hexdigest)rDr sha256_hash byte_blockr:rr;r s   z'MinaionsRFPService._calculate_file_hash)rr)NT)rQT)rQTN)TT)2r,r-r.r/rfrErTrrrUrrrrr4rlrrrrrrr rr%rrTrrr[rrrrrrrrrrr rr5rr<rIr@r?rrr:r:r:r;rs&Gx%"0)$.4=   ^x$          T   '"  4 mG7  ) " bT *,r__main__);rrPrloggingrpathlibrtypingrrrrrr rrMrr tempfilerrrrbotocore.exceptionsr log_forwarderr bid_prep_automationr_ bid_queriesrr rtender_automationrdocument_extractorr rextract_annexures_autor\rAmain_document_processorr&dotenvrgetenvrrr basicConfigINFO getLoggerr,r_rr+r<rservicer:r:r:r;s4                hE a