fix: merging dev changes to main branch (#1786)

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: Ajit Padhi <v-padhiajit@microsoft.com>
Co-authored-by: Pavan-Microsoft <v-kupavan@microsoft.com>
Co-authored-by: Ross Smith <ross-p-smith@users.noreply.github.com>
Co-authored-by: gpickett <122489228+gpickett@users.noreply.github.com>
Co-authored-by: Francia Riesco <friesco@microsoft.com>
Co-authored-by: Francia Riesco <Fr4nc3@users.noreply.github.com>
Co-authored-by: Prajwal D C <v-dcprajwal@microsoft.com>
Co-authored-by: Harmanpreet-Microsoft <v-harmanprka@microsoft.com>
Co-authored-by: UtkarshMishra-Microsoft <v-utkamishra@microsoft.com>
Co-authored-by: Priyanka-Microsoft <v-prisinghal@microsoft.com>
Co-authored-by: Prasanjeet-Microsoft <v-singhprasa@microsoft.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Kiran-Siluveru-Microsoft <v-ksiluveru@microsoft.com>
Co-authored-by: Prashant-Microsoft <v-pmalusare@microsoft.com>
Co-authored-by: Rohini-Microsoft <v-rwalunj@microsoft.com>
Co-authored-by: Avijit-Microsoft <v-aghorui@microsoft.com>
Co-authored-by: RaviKiran-Microsoft <v-ravikirans@microsoft.com>
Co-authored-by: Somesh Joshi <v-somejoshi@microsoft.com>
Co-authored-by: Himanshi Agrawal <v-himagrawal@microsoft.com>
Co-authored-by: pradeepjha-microsoft <v-pradeepjha@microsoft.com>
Co-authored-by: Harmanpreet Kaur <v-harmanpkau@microsoft.com>
Co-authored-by: Bangarraju-Microsoft <v-golib@microsoft.com>
Co-authored-by: Harsh-Microsoft <v-hbangera@microsoft.com>
Co-authored-by: Kanchan-Microsoft <v-knagshetti@microsoft.com>
Co-authored-by: Cristopher Coronado <cristofima@hotmail.com>
Co-authored-by: Cristopher Coronado Moreira <crcorona@pichincha.com>

Author: 26 people

.github/workflows/build-docker.yml

@@ -56,7 +56,7 @@ jobs:
 
     # Login for 'dev' and 'demo' branches to cwydcontainerreg only
     - name: Docker Login to cwydcontainerreg (Dev/Demo)
-      if: ${{ inputs.push == true && (github.ref_name == 'dev' || github.ref_name == 'demo') }}
+      if: ${{ inputs.push == true && (github.ref_name == 'dev' || github.ref_name == 'demo' || github.ref_name == 'dependabotchanges') }}
       uses: docker/login-action@v3
       with:
         registry: ${{ inputs.new_registry }}

code/backend/batch/utilities/document_chunking/chunking_strategy.py

@@ -6,6 +6,7 @@ class ChunkingStrategy(Enum):
     PAGE = "page"
     FIXED_SIZE_OVERLAP = "fixed_size_overlap"
     PARAGRAPH = "paragraph"
+    JSON = "json"
 
 
 class ChunkingSettings:

code/backend/batch/utilities/document_chunking/json.py

@@ -0,0 +1,37 @@
+import json
+from typing import List
+from .document_chunking_base import DocumentChunkingBase
+from langchain.text_splitter import RecursiveJsonSplitter
+from .chunking_strategy import ChunkingSettings
+from ..common.source_document import SourceDocument
+
+
+class JSONDocumentChunking(DocumentChunkingBase):
+    def __init__(self) -> None:
+        pass
+
+    def chunk(
+        self, documents: List[SourceDocument], chunking: ChunkingSettings
+    ) -> List[SourceDocument]:
+        full_document_content = "".join(
+            list(map(lambda document: str(document.content), documents))
+        )
+        document_url = documents[0].source
+        json_data = json.loads(full_document_content)
+        splitter = RecursiveJsonSplitter(max_chunk_size=chunking.chunk_size)
+        chunked_content_list = splitter.split_json(json_data)
+        # Create document for each chunk
+        documents = []
+        chunk_offset = 0
+        for idx, chunked_content in enumerate(chunked_content_list):
+            documents.append(
+                SourceDocument.from_metadata(
+                    content=str(chunked_content),
+                    document_url=document_url,
+                    metadata={"offset": chunk_offset},
+                    idx=idx,
+                )
+            )
+
+            chunk_offset += len(chunked_content)
+        return documents

code/backend/batch/utilities/document_chunking/strategies.py

@@ -3,6 +3,7 @@
 from .page import PageDocumentChunking
 from .fixed_size_overlap import FixedSizeOverlapDocumentChunking
 from .paragraph import ParagraphDocumentChunking
+from .json import JSONDocumentChunking
 
 
 def get_document_chunker(chunking_strategy: str):
@@ -14,5 +15,7 @@ def get_document_chunker(chunking_strategy: str):
         return FixedSizeOverlapDocumentChunking()
     elif chunking_strategy == ChunkingStrategy.PARAGRAPH.value:
         return ParagraphDocumentChunking()
+    elif chunking_strategy == ChunkingStrategy.JSON.value:
+        return JSONDocumentChunking()
     else:
         raise Exception(f"Unknown chunking strategy: {chunking_strategy}")

code/backend/batch/utilities/helpers/config/config_helper.py

@@ -68,6 +68,7 @@ def get_available_document_types(self) -> list[str]:
             "jpg",
             "png",
             "docx",
+            "json"
         }
         if self.env_helper.USE_ADVANCED_IMAGE_PROCESSING:
             document_types.update(ADVANCED_IMAGE_PROCESSING_FILE_TYPES)
@@ -308,10 +309,10 @@ def clear_config():
     @staticmethod
     def _append_advanced_image_processors():
         image_file_types = ["jpeg", "jpg", "png", "tiff", "bmp"]
-        ConfigHelper._remove_processors_for_file_types(image_file_types)
+        # ConfigHelper._remove_processors_for_file_types(image_file_types)
         ConfigHelper._default_config["document_processors"].extend(
             [
-                {"document_type": file_type, "use_advanced_image_processing": True}
+                {"document_type": file_type, "chunking" : ConfigHelper._default_config["document_processors"][0]["chunking"], "loading" : ConfigHelper._default_config["document_processors"][0]["loading"], "use_advanced_image_processing": True}
                 for file_type in image_file_types
             ]
         )

code/backend/batch/utilities/helpers/config/default.json

@@ -97,6 +97,17 @@
         "strategy": "docx"
       }
     },
+    {
+      "document_type": "json",
+      "chunking": {
+        "strategy": "json",
+        "size": 500,
+        "overlap": 100
+      },
+      "loading": {
+        "strategy": "web"
+      }
+    },
     {
       "document_type": "jpg",
       "chunking": {