o L%i@sNdZddlZddlZddlZddlZddlmZddlmZddlm Z m Z m Z m Z ddl Z ddlZddlZddlZddlmZddlZddlZddlZddlmZedZed d d Zd Zd ZeddZedZedZejejde de!gde"e#Z$GdddZ%ddZ&e#dkre&ddZ&e#dkre&dSdS)z Bid Processing Script - Extract and process government tender data Reads PDF work orders and JSON files, uses Claude LLM for PDF extraction N)datetime)Path)DictListOptionalAny)OpenAI) load_dotenvzclaude-haiku-4-5 gpZ6FfGe2DKNfIIM2hzqIctuVnntHgvSz#https://api.deepinfra.com/v1/openai)api_keybase_urlopen_llmz Qwen/Qwen3-Next-80B-A3B-InstructAPI_URLzhttp://localhost:5000INTERNAL_API_KEYMASTER_TENANT_IDz)%(asctime)s - %(levelname)s - %(message)szbid_processing.log)levelformathandlersc @s.eZdZ  d.dededefddZdefd d Zd ed edeefd dZ d/ded ed ededeef ddZ d/ded ede ede efddZ ddZ dedefddZdededefddZdedefd d!Zd/ded"eedeefd#d$Z d/d%ed&ed'ed(ed)e edef d*d+Zd,d-ZdS)0 BidProcessorprocessed_bids_data.jsonNroot_directory output_fileanthropic_api_keycCsPt||_t||_|ptd|_|jstdtj|jd|_ | |_ dS)z Initialize the bid processor Args: root_directory: Root directory containing bid folders output_file: Output JSON file path anthropic_api_key: Anthropic API key for Claude ANTHROPIC_API_KEYz[Anthropic API key required. Set ANTHROPIC_API_KEY environment variable or pass it directly.)r N) rrrosgetenvr ValueError anthropic Anthropicclient_load_existing_dataprocessed_data)selfrrrr#=/var/www/html/minaions-tender/ai-engine/process_eproc_data.py__init__9s zBidProcessor.__init__returnc Cs|jrOz/t|jddd }t|}Wdn1swYtdt|dgd|WSt yN}zt d|WYd}~nd}~wwt d d d d d d gd S)z)Load existing processed data if availablerutf-8encodingNzLoaded existing data with bidsz bidszError loading existing data: rzGovernment Tender Processingz0.0%) scrapedAt totalBids totalFoundsourcewithDetailedInfodetailFetchSuccess)metadatar+)rexistsopenjsonloadloggerinfolenget Exceptionerrorrnow isoformat)r"fdataer#r#r$r Ms(   z BidProcessor._load_existing_data file_path bid_numberc Csrz|std|WdStrtsgtdtdtdts+tdts_tdtdtdtd td td tdtd td tdtdWdS|dddd}|j}dtd|d|}ddddddddddddd }|j }| |d}t |do} d || |fi} t||d!d"} d#ti} t j td$| | | d%d&} | jd'kr| }| d(rtd)|| d*WdWStd+| d, WdWdStd-| jd.| j WdWdS1swYWdSty8}ztd/|jd0|WYd}~dSd}~ww)1z Upload file to backend via API (stores in S3) Args: file_path: Path to file bid_number: Bid number for organizing files Returns: Dict with storage details or None if failed zFile not found: NzP================================================================================zHCONFIGURATION ERROR: INTERNAL_API_KEY or MASTER_TENANT_ID not configuredu.❌ INTERNAL_API_KEY is missing from .env fileu.❌ MASTER_TENANT_ID is missing from .env filez To fix this:z.1. Connect to MongoDB and find your tenant ID:z mongo> use tenderdbz: mongo> db.tenants.findOne({}, {_id: 1, companyName: 1})z(2. Add the tenant _id to ai-engine/.env:z) MASTER_TENANT_ID=/_\znexray_training/application/pdfzapplication/mswordzGapplication/vnd.openxmlformats-officedocument.wordprocessingml.documentzapplication/vnd.ms-excelzAapplication/vnd.openxmlformats-officedocument.spreadsheetml.sheetz text/plainapplication/jsonzapplication/zipzapplication/x-rar-compressedz image/pngz image/jpeg) .pdf.doc.docx.xls.xlsxz.txtz.jsonz.zipz.rarz.pngz.jpgz.jpegzapplication/octet-streamrbfilefalse) tenant_idkey content_typeencryptx-internal-api-keyz/internal-api/storage/upload<)filesr@headerstimeoutsuccessu✅ Uploaded file to S3: r@zBackend upload failed: messagezUpload API error:  - zError uploading file : )r3r7warningrrr<replacenamesuffixlowerr:r4requestspostBACKEND_API_URL status_coder5r8textr;)r"rBrCsafe_bid_number file_namefile_keycontent_type_mapfile_extrTr?rXr@rYresponseresultrAr#r#r$_upload_file_to_backendes                     * z$BidProcessor._upload_file_to_backendstorage_detailsr2c Cs.z{|sWdS|j}|}d}t||dd||d|tdd|p#id}td d } tjt d || d d } | j dvrl| } | dr^| di d} t d|d| d| WSt d| dWdSt d| j d| jWdSty} zt d| WYd} ~ dSd} ~ ww)a: Create Document record in database via API Args: storage_details: Storage details from S3 upload file_path: Original file path bid_number: Bid number metadata: Additional metadata Returns: Document ID or None if failed Nnexray_documentnexray_trainings3eproc_historicalT) bidNumberr/originalFileName uploadedAtisNexRayTraining)tenantrbtypecategory storageTypestorageDetailsr2rI)rVz Content-Typez/internal-api/documents)r5rYrZ)r[r\r@_idu✅ Created document record: z (ID: )zDocument creation failed: r]zDocument API error: r^z Error creating document record: )rbrdrrr=r>rrerfrgrhr5r:r7r8r<rir;)r"rrrBrCr2rk file_lowerdoc_type document_datarYrorp document_idrAr#r#r$_create_document_recordsZ   z$BidProcessor._create_document_record folder_pathexisting_documentsc Csg}|rt|dkrtd|d|Szhd}g}|D]}|r2|j|vr2||q |s@td||WStdt|d|j |D]f}zF| ||}|r|j |||d|j id } | r|| |j | d | d | d t d ntd|j n td|j WqPty} ztd|j d| WYd} ~ qPd} ~ wwW|Sty} ztd| WYd} ~ |Sd} ~ ww)ai Find all files in folder, upload to S3, and create document records Args: folder_path: Path to folder containing files bid_number: Bid number for organization existing_documents: List of already uploaded documents to avoid duplicates Returns: List of document information dicts rz Documents already exist for bid z, skipping upload>rKrJrMrLrNz9No uploadable files (pdf, doc, docx, xls, xlsx) found in Found z uploadable file(s) in folderName)r2urlrSsize) documentIdfileName storageUrl storageKeyfileSizeryz%Failed to create document record for zFailed to upload zError processing file r_Nz"Error processing files in folder: )r9r7r8iterdiris_filercrdappendr`rbrqrr:rr=r>r;r<) r"rrCr documentsallowed_extensions all_filesrPrrrrAr#r#r$_process_and_upload_files sb        #z&BidProcessor._process_and_upload_filesc Cszgt|jd|jdd<tdd|jdD|jdd<|jddtd|jdddd d |jdd <t|jd d d}tj|j|dddWdn1sWwYt d|jWdSt y}zt d|WYd}~dSd}~ww)zSave processed data to diskr+r2r-css|] }d|vrdVqdS) detailedInfoNr#).0bidr#r#r$ Wsz*BidProcessor._save_data..r0rdz.1f%r1wr(r)F)indent ensure_asciiNzData saved to zError saving data: ) r9r!summaxr4rr5dumpr7r8r;r<)r"r?rAr#r#r$ _save_dataRs, zBidProcessor._save_datapdf_pathc Cszd}z:t|}|jD]}|}|r||d7}q Wdn1s&wY|r|d ddd d n|d dg|d dg|d dd gt|dkrYdndg|d} d |vrj|d g| d<|d|t|dk||dddtt|dgd|ddddddd|ddvr|ddddndd|ddvr|ddddndddg| dr| ddndt|dddd|didd d!d"} g} t|dD]\} }|d#d|d$d%d&|d$dvrd&ndd'}|d#t|vr | D]L}|d#|d#krX|d(d)|d*<d+|d,<|d#d| d-d.d/d0<|drG|d| d-d.d/d1<| t|d(d)d2dnq | d-d.d3|q| rt| | d-d.d/d4<t | | d-d.d/d5<t | t| | d-d.d/d6<|d7r| d8r| d8d9|d77<n d:|d7| d8<| | d;<| S)sz4BidProcessor.process_all_folders..rz folders to processrrNr+rrDrrzFolder z4 already fully processed with documents, skipping...z( exists but incomplete, re-processing...zUpdated existing bid data for rz Processed rEz folderszError processing folder r_zProcessing complete. Processed z new folders.)r7r8rrr9rbr%r!r:rrrtimesleepr;r<)r"folders total_foldersprocessed_countfolder folder_idrexisting_bid_indexidxrhas_detailed_info has_documentsr,rAr#r#r$process_all_folderssR    z BidProcessor.process_all_folders)rN)N)__name__ __module__ __qualname__r#r%rr rrrqrrrrrrrrrr?r#r#r#r$r8sB  `  $FH*q 8  mrc Cstjdd}|jddd|jdddd |}|j}|j}tjd }tj |s5t d |d d Sz#t |||}| }t dt dt|dt d|Wd Styz}ztd|t d|WYd }~d Sd }~ww)&Main function to run the bid processorzProcess bid folders) descriptionrz0Root directory containing bid folders to process)helpz--outputrz4Output file name (default: processed_bids_data.json))defaultrErError: Directory  does not exist!N Processing complete!Total bids processed: r+Results saved to: Error in main processing: Error: )argparseArgumentParser add_argument parse_argsroutputrenvironr:pathr3printrr?r9r;r7r<)parserargsROOT_DIRECTORY OUTPUT_FILEr processorrprAr#r#r$mains8    r[__main__c Csd}d}d}tj|std|ddSz#t|||}|}tdtdt|d td |WdSty[}zt d |td |WYd}~dSd}~ww) rCz/contentrzl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+rKrLrM) rrTr3rUrr?r9r;r7r<)rXrYrrZrprAr#r#r$r[s"  )'__doc__rr5loggingr4rpathlibrtypingrrrrrrrrrrrNredotenvr rrrrrgrr basicConfigINFO FileHandler StreamHandler getLoggerr@r7rr[r#r#r#r$sb        >(