Types and Constants¶

from rs_document import Document

class Document:
    """A document with content and metadata."""

    def __init__(
        self,
        page_content: str,
        metadata: dict[str, str]
    ) -> None:
        """
        Create a new Document.

        Args:
            page_content: The text content of the document
            metadata: Dictionary of string key-value pairs
        """
        ...

    # Attributes
    page_content: str
    metadata: dict[str, str]

    # Cleaning methods
    def clean(self) -> None:
        """Run all cleaning operations."""
        ...

    def clean_non_ascii_chars(self) -> None:
        """Remove all non-ASCII characters."""
        ...

    def clean_bullets(self) -> None:
        """Remove bullet point characters."""
        ...

    def clean_ligatures(self) -> None:
        """Convert typographic ligatures to component characters."""
        ...

    def clean_extra_whitespace(self) -> None:
        """Normalize whitespace."""
        ...

    def group_broken_paragraphs(self) -> None:
        """Join paragraphs incorrectly split across lines."""
        ...

    # Splitting methods
    def recursive_character_splitter(
        self,
        chunk_size: int
    ) -> list[Document]:
        """
        Split document with recursive strategy.

        Args:
            chunk_size: Target size for each chunk in characters

        Returns:
            List of Document instances (chunks)
        """
        ...

    def split_on_num_characters(
        self,
        num_chars: int
    ) -> list[Document]:
        """
        Split document into fixed-size chunks.

        Args:
            num_chars: Number of characters per chunk

        Returns:
            List of Document instances (chunks)
        """
        ...

from rs_document import clean_and_split_docs

def clean_and_split_docs(
    documents: list[Document],
    chunk_size: int
) -> list[Document]:
    """
    Process multiple documents in parallel: clean and split.

    Args:
        documents: List of documents to process
        chunk_size: Target size for chunks in characters

    Returns:
        Flattened list of all chunks from all documents
    """
    ...

from rs_document import Document, clean_and_split_docs

# Type checker knows the types
doc: Document = Document(
    page_content="text",
    metadata={"key": "value"}
)

# Type checker validates method calls
doc.clean()  # OK - returns None
chunks: list[Document] = doc.recursive_character_splitter(1000)  # OK

# Type checker catches errors
doc.clean_non_ascii_chars(123)  # Error: takes no arguments
bad_doc = Document(123, {})  # Error: page_content must be str

from rs_document import Document

def process_document(doc: Document) -> list[Document]:
    """Type-safe document processing."""
    doc.clean()
    return doc.recursive_character_splitter(1000)

# IDE provides autocomplete and type checking
result = process_document(my_doc)
reveal_type(result)  # list[Document]

from rs_document import Document

doc = Document(
    page_content="Paragraph 1\n\nParagraph 2\n\nParagraph 3",
    metadata={}
)

# Will try to split on "\n\n" first
chunks = doc.recursive_character_splitter(50)

# chunk_size = 1000
# overlap_size = 1000 / 3 = 333 characters

doc = Document(page_content="A" * 3000, metadata={})
chunks = doc.recursive_character_splitter(1000)

# Chunk 0: characters 0-1000
# Chunk 1: characters 667-1667 (333 char overlap with chunk 0)
# Chunk 2: characters 1334-2334 (333 char overlap with chunk 1)
# etc.

from rs_document import Document

doc = Document(page_content="", metadata={})

# Cleaning methods - no error
doc.clean()  # No-op, returns None
doc.clean_non_ascii_chars()  # No-op, returns None
doc.clean_bullets()  # No-op, returns None
doc.clean_ligatures()  # No-op, returns None
doc.clean_extra_whitespace()  # No-op, returns None
doc.group_broken_paragraphs()  # No-op, returns None

# Splitting methods - return empty list
chunks = doc.recursive_character_splitter(1000)
print(chunks)  # []

chunks = doc.split_on_num_characters(100)
print(chunks)  # []

# All strings - correct
doc = Document(
    page_content="text",
    metadata={"id": "123", "count": "456", "active": "True"}
)

# Non-string values - may cause errors
doc = Document(
    page_content="text",
    metadata={"id": 123, "active": True}  # ERROR: int and bool
)

raw_metadata = {
    "id": 123,
    "page": 5,
    "active": True,
    "score": 98.6,
    "tags": ["a", "b"],
}

# Convert all values to strings
doc = Document(
    page_content="text",
    metadata={k: str(v) for k, v in raw_metadata.items()}
)

print(doc.metadata)
# {"id": "123", "page": "5", "active": "True",
#  "score": "98.6", "tags": "['a', 'b']"}

from rs_document import Document

doc = Document(page_content="text", metadata={})

# Invalid chunk_size - negative
try:
    chunks = doc.recursive_character_splitter(-100)
except ValueError as e:
    print(f"Error: {e}")

# Invalid chunk_size - zero
try:
    chunks = doc.split_on_num_characters(0)
except ValueError as e:
    print(f"Error: {e}")

# Wrong type for chunk_size
try:
    chunks = doc.recursive_character_splitter("1000")  # string instead of int
except TypeError as e:
    print(f"Error: {e}")

from rs_document import clean_and_split_docs

# Empty list - returns empty list
chunks = clean_and_split_docs([], chunk_size=1000)
print(chunks)  # []

# Cleaning methods - no parameters
doc.clean()
doc.clean_non_ascii_chars()
doc.clean_bullets()
doc.clean_ligatures()
doc.clean_extra_whitespace()
doc.group_broken_paragraphs()

# Splitting methods - required parameters
chunks = doc.recursive_character_splitter(chunk_size=1000)  # chunk_size required
chunks = doc.split_on_num_characters(num_chars=500)  # num_chars required

# Utility function - required parameters
from rs_document import clean_and_split_docs
chunks = clean_and_split_docs(documents, chunk_size=1000)  # both required

# Python 3.10+ syntax (used in rs_document)
def process(docs: list[Document]) -> dict[str, str]:
    ...

# Older syntax (also supported)
from typing import List, Dict
def process(docs: List[Document]) -> Dict[str, str]:
    ...

# Both use same attribute names
from rs_document import Document as RSDocument
from langchain_core.documents import Document as LCDocument

rs_doc = RSDocument(page_content="text", metadata={"key": "value"})
lc_doc = LCDocument(page_content="text", metadata={"key": "value"})

# Both have same attributes
print(rs_doc.page_content)  # "text"
print(lc_doc.page_content)  # "text"

print(rs_doc.metadata)  # {"key": "value"}
print(lc_doc.metadata)  # {"key": "value"}

from langchain_core.documents import Document as LCDocument
from rs_document import Document as RSDocument

lc_doc = LCDocument(
    page_content="text",
    metadata={"id": 123, "active": True, "tags": ["a", "b"]}
)

# Convert - stringify all metadata
rs_doc = RSDocument(
    page_content=lc_doc.page_content,
    metadata={k: str(v) for k, v in lc_doc.metadata.items()}
)

from rs_document import Document as RSDocument
from langchain_core.documents import Document as LCDocument

rs_doc = RSDocument(
    page_content="text",
    metadata={"id": "123", "page": "5"}
)

# Convert - direct copy (already strings)
lc_doc = LCDocument(
    page_content=rs_doc.page_content,
    metadata=rs_doc.metadata
)

from langchain_core.documents import Document as LCDocument
from rs_document import Document as RSDocument, clean_and_split_docs

# Start with LangChain documents
lc_documents = [
    LCDocument(page_content=text, metadata={"source": f"doc{i}.txt"})
    for i, text in enumerate(texts)
]

# Convert to rs_document for processing
rs_documents = [
    RSDocument(
        page_content=doc.page_content,
        metadata={k: str(v) for k, v in doc.metadata.items()}
    )
    for doc in lc_documents
]

# Process with rs_document (fast)
chunks = clean_and_split_docs(rs_documents, chunk_size=1000)

# Convert back to LangChain
lc_chunks = [
    LCDocument(
        page_content=chunk.page_content,
        metadata=chunk.metadata
    )
    for chunk in chunks
]

# Now use with LangChain tools
# vectorstore.add_documents(lc_chunks)