Skip to content

fix(text): complete Adobe Glyph List mapping in glyph_name_to_unicode for /Differences font encodings #572

Description

@oshtivi

What happened?

In oxidize-pdf-core/src/text/extraction_cmap.rs, glyph_name_to_unicode only contains mappings for digits 0..=9, symbols, and uppercase 'A'..='C'. It returns None for all lowercase letters ('a'..='z'), uppercase 'D'..='Z', and standard Adobe Glyph List (AGL) names (germandbls, periodcentered, bullet, hyphen, etc.).

When a PDF contains a Type 1 or TrueType font with custom /Differences in its /Encoding dictionary (and no /ToUnicode CMap), unmapped glyph names fall back to decode_winansi(byte). Because the character codes in custom-encoded fonts often use low indices (\x01, \x02, \x03...), they decode as ASCII control characters and are subsequently stripped by sanitize_extracted_text, resulting in missing or corrupted words.

Minimal reproduction

// Cargo.toml dependencies:
// oxidize-pdf = "5.0.1"
// lopdf = "0.34.0"

use lopdf::content::{Content, Operation};
use lopdf::{Document, Object, Stream, dictionary};
use oxidize_pdf::parser::PdfReader;
use oxidize_pdf::text::TextExtractor;

fn generate_type1_differences_pdf() -> Vec<u8> {
    let mut doc = Document::with_version("1.5");
    let pages_id = doc.new_object_id();

    // /Encoding dictionary mapping bytes 1..5 to glyph names /plus, /T, /e, /s, /t
    let enc_dict = dictionary! {
        "Type" => "Encoding",
        "BaseEncoding" => "WinAnsiEncoding",
        "Differences" => Object::Array(vec![
            Object::Integer(1),
            Object::Name(b"plus".to_vec()),
            Object::Name(b"T".to_vec()),
            Object::Name(b"e".to_vec()),
            Object::Name(b"s".to_vec()),
            Object::Name(b"t".to_vec()),
        ]),
    };
    let enc_id = doc.add_object(enc_dict);
    let font_id = doc.add_object(dictionary! {
        "Type" => "Font",
        "Subtype" => "Type1",
        "BaseFont" => "Helvetica",
        "Encoding" => enc_id,
    });
    let res_id = doc.add_object(dictionary! {
        "Font" => dictionary! { "F1" => font_id },
    });

    // Content stream showing byte sequence [1, 2, 3, 4, 5]
    let content = Content {
        operations: vec![
            Operation::new("BT", vec![]),
            Operation::new("Tf", vec![Object::Name(b"F1".to_vec()), Object::Integer(12)]),
            Operation::new("Tj", vec![Object::String(vec![1, 2, 3, 4, 5], lopdf::StringFormat::Literal)]),
            Operation::new("ET", vec![]),
        ],
    };
    let content_id = doc.add_object(Stream::new(dictionary! {}, content.encode().unwrap()));
    let page_id = doc.add_object(dictionary! {
        "Type" => "Page",
        "Parent" => pages_id,
        "Contents" => content_id,
        "Resources" => res_id,
    });
    doc.objects.insert(pages_id, Object::Dictionary(dictionary! {
        "Type" => "Pages",
        "Kids" => Object::Array(vec![Object::Reference(page_id)]),
        "Count" => 1_i64,
        "MediaBox" => Object::Array(vec![0.into(), 0.into(), 595.into(), 842.into()]),
    }));
    let cat_id = doc.add_object(dictionary! { "Type" => "Catalog", "Pages" => pages_id });
    doc.trailer.set("Root", cat_id);

    let mut bytes = Vec::new();
    doc.save_to(&mut bytes).unwrap();
    bytes
}

fn main() {
    let pdf_bytes = generate_type1_differences_pdf();
    let reader = PdfReader::new(std::io::Cursor::new(pdf_bytes)).unwrap();
    let doc = reader.into_document();

    let mut extractor = TextExtractor::new();
    let page = extractor.extract_from_page(&doc, 0).unwrap();

    println!("Extracted text: {:?}", page.text);
    // Expected: "+Test"
    // Actual:   "+" (because 'e', 's', 't' are not mapped in glyph_name_to_unicode and get stripped)
    assert_eq!(page.text.trim(), "+Test");
}

oxidize-pdf version

5.0.1

Rust version

1.96.0

Operating system

macOS / Linux

Activity

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Projects

    No projects

      Milestone

      No milestone

      Relationships

      None yet

      Development

      No branches or pull requests

      Issue actions