<?php
/**
 * @author 1902 Software
 * @copyright Copyright © 2023 1902 Software (https://1902software.com/magento/)
 * @package WriteTextAI_WriteTextAI
 */

namespace WriteTextAI\WriteTextAI\Model;

use Magento\Framework\App\Config\ScopeConfigInterface;
use Magento\Store\Model\ScopeInterface;
use WriteTextAI\WriteTextAI\Model\ApiManager;
use WriteTextAI\WriteTextAI\Helper\Store as StoreHelper;
use WriteTextAI\WriteTextAI\Model\OptionSource\Filter\Fields;
use WriteTextAI\WriteTextAI\Helper\Data as HelperData;
use WriteTextAI\WriteTextAI\Model\AiProductManager;
use WriteTextAI\WriteTextAI\Model\GeneratedText;
use WriteTextAI\WriteTextAI\Model\GeneratedTextOptimized;
use WriteTextAI\WriteTextAI\Model\ReviewStatus;
use WriteTextAI\WriteTextAI\Helper\Product as ProductHelper;
use WriteTextAI\WriteTextAI\Helper\Categories\Fields as CategoryFields;
use Magento\Catalog\Model\ResourceModel\Product\Action as ProductAction;
use Magento\Catalog\Api\ProductRepositoryInterface;
use Magento\Store\Model\Store;
use Magento\Framework\Stdlib\DateTime\TimezoneInterface;
use Magento\Catalog\Model\ResourceModel\Product\Gallery as ProductGallery;
use Magento\Catalog\Model\CategoryFactory;
use WriteTextAI\WriteTextAI\Model\Categories\GeneratedText as CategoryGeneratedText;
use WriteTextAI\WriteTextAI\Model\AiCategoryManager;
use WriteTextAI\WriteTextAI\Helper\Html;
use WriteTextAI\WriteTextAI\Helper\Categories\RepresentativeProducts as RepresentativeProductsHelper;
use WriteTextAI\WriteTextAI\ViewModel\Premium;
use WriteTextAI\WriteTextAI\Model\ArchiveManager;
use WriteTextAI\WriteTextAI\Logger\CronLogger;

class FullAutomationCronManager
{
    /**
     * Batch size for processing records
     */
    const BATCH_SIZE = 50; // Process 50 records at a time
    /**
     * Delay between batches in microseconds (0.5 seconds)
     */
    const BATCH_DELAY = 500000; // 0.5 seconds between batches
    /**
     * Maximum retries for failed API calls
     */
    const MAX_RETRIES = 3; // Maximum retry attempts for API calls
    /**
     * Memory limit in MB before forcing cleanup
     */
    const MEMORY_LIMIT = 256; // Memory limit in MB before forcing cleanup
    /**
     * Maximum execution time in seconds (1 hour)
     */
    const MAX_EXECUTION_TIME = 3600; // Maximum execution time in seconds (1 hour)
    
    /**
     * @var ScopeConfigInterface
     */
    protected $scopeConfig;

    /**
     * @var ApiManager
     */
    protected $apiManager;

    /**
     * @var StoreHelper
     */
    protected $storeHelper;

    /**
     * @var HelperData
     */
    protected $helperData;

    /**
     * @var AiProductManager
     */
    protected $aiProductManager;

    /**
     * @var GeneratedText
     */
    protected $generatedText;

    /**
     * @var GeneratedTextOptimized
     */
    protected $generatedTextOptimized;

    /**
     * @var ReviewStatus
     */
    protected $reviewStatus;

    /**
     * @var ProductHelper
     */
    protected $productHelper;

    /**
     * @var ProductAction
     */
    protected $productAction;

    /**
     * @var ProductRepositoryInterface
     */
    protected $productRepository;

    /**
     * @var TimezoneInterface
     */
    protected $timezone;

    /**
     * @var ProductGallery
     */
    protected $productGalleryResource;

    /**
     * @var CategoryFactory
     */
    protected $categoryFactory;

    /**
     * @var CategoryFields
     */
    protected $categoryFields;

    /**
     * @var CategoryGeneratedText
     */
    protected $categoryGeneratedText;

    /**
     * @var AiCategoryManager
     */
    protected $aiCategoryManager;

    /**
     * @var Html
     */
    protected $htmlHelper;

    /**
     * @var RepresentativeProductsHelper
     */
    protected $representativeProductsHelper;

    /**
     * @var CronLogger
     */
    protected $cronLogger;

    /**
     * @var Premium
     */
    protected $premium;

    /**
     * @var array
     */
    protected $transferResults = [];

    /**
     * @var array
     */
    protected $failedRecords = [];

    /**
     * @var array
     */
    protected $processedRecords = [];

    /**
     * @param ScopeConfigInterface $scopeConfig
     * @param ApiManager $apiManager
     * @param StoreHelper $storeHelper
     * @param HelperData $helperData
     * @param AiProductManager $aiProductManager
     * @param GeneratedText $generatedText
     * @param ReviewStatus $reviewStatus
     * @param ProductHelper $productHelper
     * @param ProductAction $productAction
     * @param ProductRepositoryInterface $productRepository
     * @param TimezoneInterface $timezone
     * @param ProductGallery $productGalleryResource
     * @param CategoryFactory $categoryFactory
     * @param CategoryFields $categoryFields
     * @param CategoryGeneratedText $categoryGeneratedText
     * @param AiCategoryManager $aiCategoryManager
     * @param Html $htmlHelper
     * @param RepresentativeProductsHelper $representativeProductsHelper
     * @param Premium $premium
     * @param CronLogger $cronLogger
     */
    public function __construct(
        ScopeConfigInterface $scopeConfig,
        ApiManager $apiManager,
        StoreHelper $storeHelper,
        HelperData $helperData,
        AiProductManager $aiProductManager,
        GeneratedText $generatedText,
        GeneratedTextOptimized $generatedTextOptimized,
        ReviewStatus $reviewStatus,
        ProductHelper $productHelper,
        ProductAction $productAction,
        ProductRepositoryInterface $productRepository,
        TimezoneInterface $timezone,
        ProductGallery $productGalleryResource,
        CategoryFactory $categoryFactory,
        CategoryFields $categoryFields,
        CategoryGeneratedText $categoryGeneratedText,
        AiCategoryManager $aiCategoryManager,
        Html $htmlHelper,
        RepresentativeProductsHelper $representativeProductsHelper,
        Premium $premium,
        CronLogger $cronLogger
    ) {
        $this->scopeConfig = $scopeConfig;
        $this->apiManager = $apiManager;
        $this->storeHelper = $storeHelper;
        $this->helperData = $helperData;
        $this->aiProductManager = $aiProductManager;
        $this->generatedText = $generatedText;
        $this->generatedTextOptimized = $generatedTextOptimized;
        $this->reviewStatus = $reviewStatus;
        $this->productHelper = $productHelper;
        $this->productAction = $productAction;
        $this->productRepository = $productRepository;
        $this->timezone = $timezone;
        $this->productGalleryResource = $productGalleryResource;
        $this->categoryFactory = $categoryFactory;
        $this->categoryFields = $categoryFields;
        $this->categoryGeneratedText = $categoryGeneratedText;
        $this->aiCategoryManager = $aiCategoryManager;
        $this->htmlHelper = $htmlHelper;
        $this->representativeProductsHelper = $representativeProductsHelper;
        $this->premium = $premium;
        $this->cronLogger = $cronLogger;
    }

    /**
     * Process full automation for entity type
     *
     * @param string $entityType
     * @return int
     */
    public function processFullAutomation($entityType)
    {
        $this->cronLogger->logRequest('Starting processFullAutomation for entity type: ' . $entityType);
        $this->cronLogger->logRequest("Initial memory usage: " . memory_get_usage(true) / 1024 / 1024 . " MB");
        
        // Initialize error tracking
        $totalErrors = 0;
        $maxErrors = 10; // Stop if we encounter too many errors
        
        if (!$this->premium->getHasProAccess()) {
            return 0;
        }

        try {
            // Clear any existing PHP cURL DNS cache
            if (function_exists('curl_reset')) {
                curl_reset(curl_init());
            }
            
            $currDateTime = $this->timezone->date()->format(\Magento\Framework\Stdlib\DateTime::DATETIME_PHP_FORMAT);
            $stores = $this->storeHelper->getAllStores(true); //true to include default store
            $fieldsConfig = $this->getAutomationFields($entityType);
            
            if (empty($fieldsConfig) || empty($fieldsConfig['fields'])) {
                $this->cronLogger->logRequest('empty fields config');
                return 0;
            }
            
            // Initialize transfer results tracking for this entity type
            $this->initializeTransferResults($entityType);
            
            $totalProcessedRecords = 0;
            
            // Process data per store
            foreach ($stores as $store) {
                $storeId = $store->getId();
                $language = $this->storeHelper->getFormattedLanguage($storeId);
                
                $this->cronLogger->logRequest("Processing store: " . $storeId . ' with language ' . $language);
                
                $processedRecords = 0;
                    
                if ($entityType === 'Product') {
                    // Process image alt texts if enabled
                    $processedRecords += $this->processAndTransferImageAltText($storeId, $language);
                }
                
                // Build params for API call
                $params = [
                    "type" => $entityType,
                    "storeId" => $storeId,
                    "language" => $language,
                    "fields" => $fieldsConfig['fields'],
                    "status" => ["ForTransferring"]
                ];
                $batchCount = 0;
                
                try {
                    // Fetch data from API with retry logic
                    $result = $this->apiCallWithRetry(function() use ($params) {
                        return $this->apiManager->getGeneratedStatus($params);
                    });
                    
                    if ($result === null) {
                        $this->cronLogger->logRequest("Failed to fetch initial data for store {$storeId} after retries");
                        $totalErrors++;
                        if ($totalErrors >= $maxErrors) {
                            $this->cronLogger->logRequest('Too many errors encountered, stopping process');
                            break;
                        }
                        continue;
                    }
                    
                    $this->cronLogger->logRequest('Initial result record count: ' . (isset($result['records']) ? count($result['records']) : 0));

                    if (!empty($result['records'])) {
                        $records = $result['records'];
                        $processedRecords = $this->processAndTransferDataBatched(
                            $records,
                            $storeId,
                            $language,
                            $entityType,
                            $fieldsConfig,
                            $batchCount
                        );
                        
                        // Process pagination if continuation token exists
                        while (isset($result['continuationToken'])) {
                            // Add delay between pagination requests
                            usleep(self::BATCH_DELAY);
                            
                            // Force garbage collection periodically
                            if ($batchCount % 5 == 0) {
                                gc_collect_cycles();
                                $this->cronLogger->logRequest("Memory usage after {$batchCount} batches: " . memory_get_usage(true) / 1024 / 1024 . " MB");
                            }
                            
                            $params['continuationToken'] = $result['continuationToken'];
                            
                            $result = $this->apiCallWithRetry(function() use ($params) {
                                return $this->apiManager->getGeneratedStatus($params);
                            });
                            
                            if ($result === null) {
                                $this->cronLogger->logRequest("Failed to fetch paginated data for store {$storeId} after retries");
                                $totalErrors++;
                                if ($totalErrors >= $maxErrors) {
                                    $this->cronLogger->logRequest('Too many errors encountered, stopping process');
                                    break;
                                }
                                break;
                            }
                            
                            if (!empty($result['records'])) {
                                $records = $result['records'];
                                $processedRecords += $this->processAndTransferDataBatched(
                                    $records,
                                    $storeId,
                                    $language,
                                    $entityType,
                                    $fieldsConfig,
                                    $batchCount
                                );
                            }
                        }
                    }
                    
                    $totalProcessedRecords += $processedRecords;
                    $this->cronLogger->logRequest("Completed store {$storeId}: processed {$processedRecords} records");
                    
                } catch (\Exception $e) {
                    $this->cronLogger->logRequest("Error processing store {$storeId}: " . $e->getMessage());
                    continue;
                }
                
                // Force cleanup after each store
                gc_collect_cycles();
                usleep(self::BATCH_DELAY);
            }
            
            $this->cronLogger->logRequest("Full automation complete. Total processed: {$totalProcessedRecords} records");
            return $totalProcessedRecords;
        } catch (\Exception $e) {
            $this->cronLogger->logRequest('Error processing full automation: ' . $e->getMessage());
            return 0;
        } finally {
            // Final cleanup of all resources
            $this->forceCloseFileHandles();
            \WriteTextAI\WriteTextAI\Model\Api\CurlBuilder::cleanupPool();
            gc_collect_cycles();
        }
    }
    
    /**
     * Process and transfer the data to Magento with batching
     *
     * @param array $records
     * @param int $storeId
     * @param string $language
     * @param string $entityType
     * @param array $fieldsConfig
     * @param int &$batchCount
     * @return int
     */
    private function processAndTransferDataBatched($records, $storeId, $language, $entityType, $fieldsConfig, &$batchCount)
    {
        $chunks = array_chunk($records, self::BATCH_SIZE);
        $totalProcessed = 0;
        
        foreach ($chunks as $chunk) {
            $batchCount++;
            $totalProcessed += $this->processAndTransferData($chunk, $storeId, $language, $entityType, $fieldsConfig);
            
            // Add delay between batches to prevent resource exhaustion
            usleep(self::BATCH_DELAY);
            
            // Periodic garbage collection
            if ($batchCount % 3 == 0) {
                gc_collect_cycles();
            }
        }
        
        return $totalProcessed;
    }
    
    /**
     * Process and transfer the data to Magento
     *
     * @param array $records
     * @param int $storeId
     * @param string $language
     * @param string $entityType
     * @param array $fieldsConfig
     * @return int
     */
    private function processAndTransferData($records, $storeId, $language, $entityType, $fieldsConfig)
    {
        $this->cronLogger->logRequest('records: ' . json_encode($records, JSON_PRETTY_PRINT));
        $processedCount = 0;
        $mappingSettings = $this->helperData->getMappingSettings();
        $groupRecords = [];
        
        // Track loaded products to avoid reloading and ensure cleanup
        $loadedProducts = [];
        
        foreach ($records as $record) {
            $groupRecords[$record['recordId']][] = $record['field'];
        }
        if ($entityType === 'Category' && !empty($groupRecords)) {
            $categoryIds = array_keys($groupRecords);
            $resultgenerated = $this->categoryGeneratedText->getGeneratedByStoreId(
                implode(",", $categoryIds ?? []),
                $storeId
            );

            // Process category data
            foreach ($groupRecords as $recordId => $fields) {
                try {
                    if (empty($resultgenerated) || !isset($resultgenerated[$recordId])) {
                        continue;
                    }

                    $mappingSettings = $this->categoryFields->getMappingSettings();

                    $fieldsList = [
                        Fields::CATEGORY_PAGE_TITLE => 'page_title',
                        Fields::CATEGORY_PAGE_DESCRIPTION => 'page_description',
                        Fields::CATEGORY_DESCRIPTION => 'category_description',
                    ];
                    
                    $selectedRepresentativeProducts = $this->getSelectedRepresentativeProducts($recordId, $storeId);
                    /** get image alt texts for representative products */
                    $imageAltTexts = [];
                    foreach ($selectedRepresentativeProducts as $productData) {
                        try {
                            $product = $this->productRepository->getById($productData->value, false, $storeId, true);
                        } catch (\Exception $e) {
                            continue;
                        }

                        $productImages = $this->productHelper->getPublishedImages($product, $storeId, $language);
                        if (is_array($productImages)) {
                            $imageAltTexts = array_merge($imageAltTexts, $productImages);
                        }
                    }

                    $category = $this->categoryFactory->create()->setStoreId($storeId)->load($recordId);
                    if (!$category->getId()) {
                        continue;
                    }
                    $resultFields = [];
                    foreach ($resultgenerated[$recordId] as $field => $value) {
                        if (in_array($field, $fields)) {
                            if ($imageAltTexts) {
                                $value = $this->htmlHelper->addAltTextToContentViaPreg($value, $imageAltTexts);
                            }
                            $category->setData($mappingSettings[$fieldsList[$field]], $value);
                            $resultFields[$fieldsList[$field]] = $value;
                        }
                    }

                    $category->save();
                    $this->categoryGeneratedText->saveCategoryText(
                        $storeId,
                        (string)$recordId,
                        $resultFields,
                        true,
                        false
                    );
                    
                    $this->aiCategoryManager->saveDate($recordId, $storeId, 'transferred');
                    $this->aiCategoryManager->saveDate($recordId, $storeId, 'reviewed');

                    $this->addSuccessResult($entityType, $storeId, $recordId);

                    $processedCount++;
                } catch (\Exception $e) {
                    // Track failed transfer
                    $this->addFailedResult($entityType, $storeId, $recordId);
                    continue;
                }
            }
        }

        if ($entityType === 'Product' && !empty($groupRecords)) {
            $this->cronLogger->logRequest('Processing products');
            $productIds = array_keys($groupRecords);
            $resultgenerated = $this->generatedTextOptimized->getGeneratedByStoreId(implode(",", $productIds ?? []), $storeId);
            $this->cronLogger->logRequest('resultgenerated: ' . json_encode($resultgenerated, JSON_PRETTY_PRINT));

            foreach ($groupRecords as $recordId => $fields) {
                // Process product data
                try {
                    if (empty($resultgenerated) || !isset($resultgenerated[$recordId])) {
                        continue;
                    }
                    
                    // Use cached product if available, otherwise load and cache
                    if (!isset($loadedProducts[$recordId])) {
                        try {
                            $loadedProducts[$recordId] = $this->productRepository->getById($recordId, false, $storeId);
                        } catch (\Exception $e) {
                            $this->cronLogger->logRequest("Failed to load product {$recordId}: " . $e->getMessage());
                            continue;
                        }
                    }
                    $product = $loadedProducts[$recordId];
                    $recordId = (int) $recordId;
                    //$item = $this->generatedText->getTextFields($recordId, $storeId, true);
                    $item = $resultgenerated[$recordId] ?? [];
                    $images = $this->productHelper->getPublishedImages($product, $storeId, $language);
                    $fieldsList = [
                        Fields::PAGE_TITLE => 'page_title',
                        Fields::PAGE_DESCRIPTION => 'page_description',
                        Fields::PRODUCT_DESCRIPTION => 'product_description',
                        Fields::EXCERPT => 'short_product_description',
                        Fields::OPEN_GRAPH => 'open_graph',
                    ];
                    $productFields = $this->getProductFields($fieldsList, $fields);

                    // Transfer to Magento attributes
                    $this->transferToMagento(
                        $recordId,
                        $storeId,
                        $productFields,
                        $item,
                        $mappingSettings,
                        $images
                    );
                    
                    // Save to WriteTextAI storage
                    $this->generatedTextOptimized->saveText(
                        $storeId,
                        (string)$recordId,
                        isset($item[Fields::PAGE_TITLE]) ? $item[Fields::PAGE_TITLE] : '',
                        isset($item[Fields::PAGE_DESCRIPTION]) ? $item[Fields::PAGE_DESCRIPTION] : '',
                        isset($item[Fields::PRODUCT_DESCRIPTION]) ? $item[Fields::PRODUCT_DESCRIPTION] : '',
                        isset($item[Fields::EXCERPT]) ? $item[Fields::EXCERPT] : '',
                        isset($item[Fields::OPEN_GRAPH]) ? $item[Fields::OPEN_GRAPH] : '',
                        $productFields,
                        false
                    );

                    // Track successful transfer
                    $this->addSuccessResult($entityType, $storeId, $recordId);
                    
                    $processedCount++;
                } catch (\Exception $e) {
                    // Track failed transfer
                    $this->addFailedResult($entityType, $storeId, $recordId);
                    $this->cronLogger->logRequest('Error transfer process: ' . $e->getMessage());
                    continue;
                } finally {
                    // Clear product from cache to free memory
                    if (isset($loadedProducts[$recordId])) {
                        unset($loadedProducts[$recordId]);
                    }
                }
            }
        }
        
        // Final cleanup of any remaining loaded products
        $loadedProducts = null;
        unset($loadedProducts);
        
        return $processedCount;
    }
    
    /**
     * Process and transfer image alt text
     *
     * @param int $storeId
     * @param string $language
     * @return int
     */
    private function processAndTransferImageAltText($storeId, $language)
    {
        $totalErrors = 0;
        $maxErrors = 10;
        $processedCount = 0;
        $batchCount = 0;
        
        try {
            $params = [
                "type" => "Product",
                "storeId" => $storeId,
                "language" => $language,
                "status" => ["ForTransferring"]
            ];
            
            $result = $this->apiCallWithRetry(function() use ($params) {
                $this->cronLogger->logRequest('Fetching image alt text status: ' . json_encode($params));
                return $this->apiManager->getImageAltTextStatus($params);
            });

            $this->cronLogger->logRequest('Fetched image alt text status: ' . json_encode($result['images']));
            
            if ($result === null) {
                $this->cronLogger->logRequest('Failed to fetch image alt text status after retries');
                return $processedCount;
            }
            
            $processedCount += $this->processAltTextImagesBatched($result, $storeId, $language, $batchCount);

            // Process pagination if continuation token exists
            while (isset($result['continuationToken'])) {
                // Add delay between pagination requests
                usleep(self::BATCH_DELAY);
                
                // Periodic garbage collection and connection cleanup
                if ($batchCount % 3 == 0) {
                    gc_collect_cycles();
                    \WriteTextAI\WriteTextAI\Model\Api\CurlBuilder::cleanupPool();
                    
                    // Force close any open file handles
                    $this->forceCloseFileHandles();
                    
                    // Check memory usage and force cleanup if needed
                    $currentMemory = memory_get_usage(true) / 1024 / 1024;
                    if ($currentMemory > self::MEMORY_LIMIT) {
                        $this->cronLogger->logRequest("Memory limit approached ({$currentMemory} MB), forcing cleanup");
                        gc_collect_cycles();
                        \WriteTextAI\WriteTextAI\Model\Api\CurlBuilder::cleanupPool();
                        $this->forceCloseFileHandles();
                    }
                }
                
                $params['continuationToken'] = $result['continuationToken'];
                
                $result = $this->apiCallWithRetry(function() use ($params) {
                    return $this->apiManager->getImageAltTextStatus($params);
                });
                
                if ($result === null) {
                    $this->cronLogger->logRequest('Failed to fetch paginated alt text data after retries');
                    $totalErrors++;
                    if ($totalErrors >= $maxErrors) {
                        $this->cronLogger->logRequest('Too many errors encountered, stopping process');
                        break;
                    }
                    break;
                }

                $processedCount += $this->processAltTextImagesBatched($result, $storeId, $language, $batchCount);
            }
        } catch (\Exception $e) {
            $this->cronLogger->logRequest('Error alt transfer process: ' . $e->getMessage());
        } finally {
            // Force cleanup of all resources
            $this->forceCloseFileHandles();
            \WriteTextAI\WriteTextAI\Model\Api\CurlBuilder::cleanupPool();
            gc_collect_cycles();
        }
        
        return $processedCount;
    }

    /**
     * Process alt text images with batching
     *
     * @param array $result
     * @param int $storeId
     * @param string $language
     * @param int &$batchCount
     * @return int
     */
    private function processAltTextImagesBatched($result, $storeId, $language, &$batchCount)
    {
        $this->cronLogger->logRequest('Processing alt text images batch: ' . json_encode($result['images']));
        $this->cronLogger->logRequest('Processing alt text images batch storeId: ' . $storeId);
        $this->cronLogger->logRequest('Processing alt text images batch language: ' . $language);
        if (empty($result['images'])) {
            return 0;
        }
        
        $processedCount = 0;
        $imageChunks = array_chunk($result['images'], self::BATCH_SIZE);
        
        foreach ($imageChunks as $chunk) {
            $batchCount++;
            $processedCount += $this->processAltTextImages(['images' => $chunk], $storeId, $language);
            
            // Add delay between batches
            usleep(self::BATCH_DELAY);
            
            // Periodic garbage collection
            if ($batchCount % 3 == 0) {
                gc_collect_cycles();
            }
        }
        
        return $processedCount;
    }
    
    /**
     * Process alt text images
     *
     * @param array $result
     * @param int $storeId
     * @param string $language
     * @return int
     */
    private function processAltTextImages($result, $storeId, $language)
    {
        $processedCount = 0;
        $imageIds = [];
        if (!empty($result['images'])) {
            $imageIds = array_column($result['images'], 'imageId');
        }

        if (empty($imageIds)) {
            return 0;
        }
        
        $productIds = $this->fetchProductIdsByImageIds($imageIds, $storeId);
        $this->cronLogger->logRequest('Processing alt text images productIds: ' . json_encode($productIds));

        if (empty($productIds)) {
            //archive product image if empty product ids - batch the archive requests
            $archiveChunks = array_chunk($imageIds, 10);
            foreach ($archiveChunks as $chunk) {
                foreach ($chunk as $imageId) {
                    try {
                        $archiveBody = [
                            "imageId" => (string)$imageId,
                        ];
                        $this->apiCallWithRetry(function() use ($archiveBody) {
                            $this->cronLogger->logRequest('Archiving image ' . $imageId);
                            return $this->apiManager->archiveProductImage($archiveBody);
                        }, 2);
                    } catch (\Exception $e) {
                        $this->cronLogger->logRequest('Failed to archive image ' . $imageId . ': ' . $e->getMessage());
                    }
                }
                usleep(100000); // 0.1 second delay between archive batches
            }
            return 0;
        }
        $imagesToUpdate = [];
        $processedProductIds = [];

        try {
            foreach ($productIds as $productId) {
                $this->cronLogger->logRequest("Processing product {$productId} for store {$storeId}");
                
                $product = null;
                try {
                    $product = $this->productRepository->getById($productId, true, $storeId);
                } catch (\Exception $e) {
                    $this->cronLogger->logRequest("Failed to load product {$productId} for store {$storeId}: " . $e->getMessage());
                    continue;
                }
                
                $mediaGalleryEntries = $product->getMediaGalleryEntries();
                $this->cronLogger->logRequest("Product {$productId} has " . count($mediaGalleryEntries) . " media gallery entries");
                $updated = false;

                if (!in_array($productId, $processedProductIds)) {
                    foreach ($mediaGalleryEntries as $mediaGalleryEntry) {
                        if (in_array($mediaGalleryEntry->getId(), $imageIds)) {
                            $this->cronLogger->logRequest("Processing image {$mediaGalleryEntry->getId()} for product {$productId}");
                            
                            $getParams = [
                                "storeId" => $storeId,
                                "language" => $language,
                                "imageId" => $mediaGalleryEntry->getId()
                            ];
        
                            $altText = $this->apiCallWithRetry(function() use ($getParams) {
                                return $this->apiManager->getImage($getParams);
                            }, 2);
                            
                            if ($altText === null) {
                                $this->cronLogger->logRequest("No alt text returned for image {$mediaGalleryEntry->getId()}");
                                continue;
                            }
                            
                            $this->cronLogger->logRequest("Setting alt text for image {$mediaGalleryEntry->getId()}: " . $altText['altText']['value']);
                            $mediaGalleryEntry->setLabel($altText['altText']['value']);
                            $updated = true;
                            
                            // Add to batch update for API
                            if (isset($altText['altText']['id'])) {
                                $imagesToUpdate[] = [
                                    "storeId" => $storeId,
                                    "language" => $language,
                                    "imageId" => $mediaGalleryEntry->getId(),
                                    "textId" => $altText['altText']['id'],
                                    "value" => $altText['altText']['value'],
                                    "publish" => true
                                ];
                            }
                        }
                    }

                    if ($updated) {
                        $this->cronLogger->logRequest("Saving updated product {$productId} for store {$storeId}");
                        $product->setMediaGalleryEntries($mediaGalleryEntries);
                        $this->productRepository->save($product);
                    
                        $this->aiProductManager->saveDate($productId, $storeId, 'transferred');
                        $this->aiProductManager->saveDate($productId, $storeId, 'reviewed');
                        
                        // Track successful transfer
                        $this->addSuccessResult('Product', $storeId, $productId);
                        $processedCount++;
                        $this->cronLogger->logRequest("Successfully processed product {$productId} for store {$storeId}");
                    } else {
                        $this->cronLogger->logRequest("No updates made for product {$productId} in store {$storeId}");
                    }

                    $processedProductIds[] = $productId;
                    
                    // Explicitly clear product reference to free memory
                    $product = null;
                    unset($product);
                }
            }

            if (!empty($imagesToUpdate)) {
                // Batch update alt text in chunks to avoid overwhelming the API
                $updateChunks = array_chunk($imagesToUpdate, 20);
                foreach ($updateChunks as $chunk) {
                    $this->apiCallWithRetry(function() use ($chunk) {
                        return $this->apiManager->batchUpdateAltText(json_encode([
                            "images" => $chunk
                        ]));
                    }, 2);
                    usleep(100000); // 0.1 second delay between update batches
                }
            }
        } catch (\Exception $e) {
            $this->cronLogger->logRequest('Error alt transfer batch process: ' . $e->getMessage());
            // Track failed transfers
            foreach ($productIds as $productId) {
                $this->addFailedResult('Product', $storeId, $productId);
            }
        }
        
        return $processedCount;
    }
    
    /**
     * Transfer data to Magento attributes
     *
     * @param int $recordId
     * @param int $storeId
     * @param array $selectedFields
     * @param array $item
     * @param array $mappingSettings
     * @param array $images
     * @return void
     */
    private function transferToMagento($recordId, $storeId, $selectedFields, $item, $mappingSettings, $images)
    {
        // Convert newlines to <br> for descriptions
        // $item['product_description'] = $this->productHelper->convertNewLinesToBr($item['product_description']);
        // $item['short_product_description'] = $this->productHelper->convertNewLinesToBr(
        //  $item['short_product_description']
        // );

        $equivalentFields = [
            Fields::PAGE_TITLE => 'page_title',
            Fields::PAGE_DESCRIPTION => 'page_description',
            Fields::PRODUCT_DESCRIPTION => 'product_description',
            Fields::EXCERPT => 'short_product_description',
            Fields::OPEN_GRAPH => 'open_graph'
        ];
        
        $toUpdate = [];
        foreach ($item as $field => $value) {
            $fieldEquivalent = $equivalentFields[$field] ?? $field;
            if (isset($mappingSettings[$fieldEquivalent]) && in_array($fieldEquivalent, $selectedFields)) {
                $value = $this->htmlHelper->addAltTextToContentViaPreg($value, $images, true);
                $toUpdate[$mappingSettings[$fieldEquivalent]] = $value;
            }
        }
        if (!empty($toUpdate)) {
            $this->productAction->updateAttributes(
                [$recordId],
                $toUpdate,
                $storeId
            );
            $this->aiProductManager->saveDate($recordId, $storeId, 'transferred');
            $this->aiProductManager->saveDate($recordId, $storeId, 'reviewed');
        }
    }
    
    /**
     * API call with retry logic
     *
     * @param callable $apiCall
     * @param int $maxRetries
     * @return mixed|null
     */
    private function apiCallWithRetry($apiCall, $maxRetries = self::MAX_RETRIES)
    {
        $attempt = 0;
        $lastException = null;
        
        // Clear DNS cache at the start
        if (function_exists('dns_get_record')) {
            clearstatcache(true);
        }
        
        // Clean up any existing cURL connections
        \WriteTextAI\WriteTextAI\Model\Api\CurlBuilder::cleanupPool();
        
        while ($attempt < $maxRetries) {
            try {
                $attempt++;
                
                // Execute the API call
                $result = $apiCall();
                
                // If successful, return the result
                return $result;
                
            } catch (\Exception $e) {
                $lastException = $e;
                $errorMessage = $e->getMessage();
                
                $this->cronLogger->logRequest("API call attempt {$attempt} failed: {$errorMessage}");
                
                // Check for specific SSL/connection errors
                if (strpos($errorMessage, 'certificate verify locations') !== false ||
                    strpos($errorMessage, 'getaddrinfo() thread failed') !== false ||
                    strpos($errorMessage, 'SSL') !== false ||
                    strpos($errorMessage, 'cURL') !== false) {
                    
                    // Exponential backoff with jitter
                    $waitTime = min(pow(2, $attempt) * 1000000, 10000000); // Max 10 seconds
                    $waitTime += rand(0, 1000000); // Add up to 1 second jitter
                    
                    $this->cronLogger->logRequest("Waiting " . ($waitTime / 1000000) . " seconds before retry...");
                    usleep($waitTime);
                    
                    // Force garbage collection to free resources
                    gc_collect_cycles();
                } else if ($attempt < $maxRetries) {
                    // For other errors, use shorter delay
                    usleep(self::BATCH_DELAY);
                } else {
                    // Final attempt failed, log and break
                    break;
                }
            }
        }
        
        // All retries exhausted
        if ($lastException) {
            $this->cronLogger->logRequest("All {$maxRetries} retry attempts failed. Last error: " . $lastException->getMessage());
        }
        
        return null;
    }
    
    /**
     * Get fields for automation from configuration
     *
     * @param string $entityType
     * @return array
     */
    private function getAutomationFields($entityType)
    {
        
        $fieldsConfig = [];
        
        if ($entityType === 'Product') {
            $fieldsConfig['fields'] = [
                Fields::PAGE_TITLE,
                Fields::PAGE_DESCRIPTION,
                Fields::PRODUCT_DESCRIPTION,
                Fields::EXCERPT,
                Fields::OPEN_GRAPH
            ];
        } elseif ($entityType === 'Category') {
            $fieldsConfig['fields'] = [
                Fields::CATEGORY_PAGE_TITLE,
                Fields::CATEGORY_PAGE_DESCRIPTION,
                Fields::CATEGORY_DESCRIPTION
            ];
        }
        
        return $fieldsConfig;
    }

    /**
     * Remove empty and already transferred fields
     *
     * @param array $fieldsList
     * @param array $fields
     * @return array
     */
    private function getProductFields($fieldsList, $fields)
    {
        $productFields = [];
        foreach ($fields as $field) {
            if (isset($fieldsList[$field])) {
                $productFields[] = $fieldsList[$field];
            }
        }
        return $productFields;
    }

    /**
     * Fetch product ids by image ids
     *
     * @param array $imageIds
     * @param int $storeId
     * @return array
     */
    private function fetchProductIdsByImageIds($imageIds, $storeId)
    {
        /** @var \Magento\Framework\DB\Adapter\AdapterInterface $connection */
        $connection = $this->productGalleryResource->getConnection();

        /**
         * Always get data from default store since we need the ids and
         * possible that the image is not available on the store
         */
        $selectValue = $connection->select()
            ->from($this->productGalleryResource->getTable(ProductGallery::GALLERY_VALUE_TABLE))
            ->where('value_id IN (?)', $imageIds)
            ->where('store_id = ?', Store::DEFAULT_STORE_ID);

        $imageValueData = $connection->fetchAll($selectValue);
        
        // Log store-specific query results
        $this->cronLogger->logRequest("Store {$storeId} image query returned " . count($imageValueData) . " records for " . count($imageIds) . " image IDs");

        $entityIds = [];
        if ($imageValueData) {
            $entityIds = array_column($imageValueData, 'entity_id');
            $entityIds = array_unique($entityIds);
        }
        
        $this->cronLogger->logRequest("Final product IDs for store {$storeId}: " . json_encode($entityIds));
        return $entityIds;
}

/**
 * Initialize transfer results tracking for an entity type
 *
     * @param string $entityType
     * @return void
     */
    private function initializeTransferResults($entityType)
    {
        if (empty($this->transferResults)) {
            $this->transferResults = [
                'types' => [
                    [
                        'type' => $entityType,
                        'all' => [
                            'success' => [],
                            'failed' => []
                        ],
                        'storeIds' => []
                    ]
                ]
            ];
        } else {
            $this->transferResults['types'][] = [
                'type' => $entityType,
                'all' => [
                    'success' => [],
                    'failed' => []
                ],
                'storeIds' => []
            ];
        }
    }

    /**
     * Add a successful result to tracking
     *
     * @param string $entityType
     * @param int $storeId
     * @param string $recordId
     * @return void
     */
    private function addSuccessResult($entityType, $storeId, $recordId)
    {
        $typeIndex = $this->findTypeIndex($entityType);
        if ($typeIndex !== false) {
            // Add to all success
            if (!in_array($recordId, $this->transferResults['types'][$typeIndex]['all']['success'])) {
                $this->transferResults['types'][$typeIndex]['all']['success'][] = (string)$recordId;
            }

            // Add to store-specific success
            if (!isset($this->transferResults['types'][$typeIndex]['storeIds'][$storeId])) {
                $this->transferResults['types'][$typeIndex]['storeIds'][$storeId] = [
                    'success' => [],
                    'failed' => []
                ];
            }
            
            if (!in_array($recordId, $this->transferResults['types'][$typeIndex]['storeIds'][$storeId]['success'])) {
                $this->transferResults['types'][$typeIndex]['storeIds'][$storeId]['success'][] = (string)$recordId;
            }
        }
    }

    /**
     * Add a failed result to tracking
     *
     * @param string $entityType
     * @param int $storeId
     * @param string $recordId
     * @return void
     */
    private function addFailedResult($entityType, $storeId, $recordId)
    {
        $typeIndex = $this->findTypeIndex($entityType);
        if ($typeIndex !== false) {
            // Add to all failed
            if (!in_array($recordId, $this->transferResults['types'][$typeIndex]['all']['failed'])) {
                $this->transferResults['types'][$typeIndex]['all']['failed'][] = (string)$recordId;
            }

            // Add to store-specific failed
            if (!isset($this->transferResults['types'][$typeIndex]['storeIds'][$storeId])) {
                $this->transferResults['types'][$typeIndex]['storeIds'][$storeId] = [
                    'success' => [],
                    'failed' => []
                ];
            }
            
            if (!in_array($recordId, $this->transferResults['types'][$typeIndex]['storeIds'][$storeId]['failed'])) {
                $this->transferResults['types'][$typeIndex]['storeIds'][$storeId]['failed'][] = (string)$recordId;
            }
        }
    }

    /**
     * Find the index of an entity type in the transfer results
     *
     * @param string $entityType
     * @return int|bool
     */
    private function findTypeIndex($entityType)
    {
        foreach ($this->transferResults['types'] as $index => $type) {
            if ($type['type'] === $entityType) {
                return $index;
            }
        }
        return false;
    }

    /**
     * Report transfer results to the API
     *
     * @return void
     */
    public function reportTransferResults()
    {
        if (!empty($this->transferResults['types'])) {
            try {
                // Format the results based on the number of stores processed
                foreach ($this->transferResults['types'] as $typeIndex => $type) {
                    $storeCount = count($this->transferResults['types'][$typeIndex]['storeIds']);
                    
                    // If multiple stores were processed, keep only storeIds
                    if ($storeCount > 1) {
                        unset($this->transferResults['types'][$typeIndex]['all']);
                    } else {
                        // If only one store or no stores were processed, keep only all
                        unset($this->transferResults['types'][$typeIndex]['storeIds']);
                    }
                }
                
                // Check if there are any processed records
                $hasProcessedRecords = false;
                foreach ($this->transferResults['types'] as $type) {
                    if (isset($type['all'])) {
                        if (!empty($type['all']['success']) || !empty($type['all']['failed'])) {
                            $hasProcessedRecords = true;
                            break;
                        }
                    } elseif (isset($type['storeIds'])) {
                        foreach ($type['storeIds'] as $storeData) {
                            if (!empty($storeData['success']) || !empty($storeData['failed'])) {
                                $hasProcessedRecords = true;
                                break 2;
                            }
                        }
                    }
                }
                
                // Only send to API if there are processed records
                if ($hasProcessedRecords) {
                    $this->cronLogger->logRequest('Sending transfer results to API');
                    $this->cronLogger->logRequest('Transfer results: ' . json_encode($this->transferResults, JSON_PRETTY_PRINT));
                    $resultEmail = $this->apiManager->emailAutomaticTransferStatus($this->transferResults);
                }
            } catch (\Exception $e) {
                // Silently handle the exception
                $this->cronLogger->logRequest('Error sending transfer results to API: ' . $e->getMessage());
                return;
            }
        }
    }

    /**
     * Get selected representative products
     *
     * @param int $categoryId
     * @param int $storeId
     * @return array
     */
    private function getSelectedRepresentativeProducts($categoryId, $storeId)
    {
        $aiCategory = $this->aiCategoryManager->getCategory($categoryId, $storeId);
        $selectedRepresentativeProducts = $this->representativeProductsHelper
            ->getRepresentativeProductsSelected($aiCategory);

        return $selectedRepresentativeProducts;
    }
    
    /**
     * Force close any open file handles to prevent "too many open files" errors
     *
     * @return void
     */
    private function forceCloseFileHandles()
    {
        // Close any open streams
        if (function_exists('get_resources')) {
            $resources = get_resources('stream');
            foreach ($resources as $resource) {
                if (is_resource($resource)) {
                    $meta = stream_get_meta_data($resource);
                    // Only close file streams, not stdin/stdout/stderr
                    if (isset($meta['uri']) && !in_array($meta['uri'], ['php://stdin', 'php://stdout', 'php://stderr'])) {
                        @fclose($resource);
                    }
                }
            }
        }
        
        // Force close any cURL handles
        if (function_exists('curl_close')) {
            // This will be handled by CurlBuilder::cleanupPool() but adding as backup
        }
        
        // Clear file stat cache to release file descriptors
        clearstatcache(true);
    }
    
    /**
     * Monitor and log current file descriptor usage
     *
     * @return void
     */
    private function logFileDescriptorUsage()
    {
        if (function_exists('get_resources')) {
            $streamCount = count(get_resources('stream'));
            $curlCount = count(get_resources('curl'));
            $this->cronLogger->logRequest("Current file descriptors - Streams: {$streamCount}, cURL: {$curlCount}");
        }
    }
}
