extractor-configuration

Extractor Konfiguration

Extractor configuration bit, used to configure extractors in articles or books

Live-Editor

 
[.extractor-configuration]
[@id:12345]
[%1]
[!My first config in TOML]

==== text ====
[Plugins]

# The enabled plugins and the order in which the plugins are run within each execution phase
plugins = [
  #
  # Plugins with phases
  #
  "Init",
  "PdfExtractPageImages_ForImages",
  "PdfExtractPageImages_ForVisualization",
  "PdfExtractEmbeddedImages",
  "PdfBlockifier",
  "Runs",
  "Tables",
  "Images",
  "Chapters",
  "Metadata",
  "Paragraphs",
  # "RegexExample",
  "DocumentMerger",
  "PageVisualizationGenerator",
  "BitmarkGenerator",

  #
  # Helper plugins, not part of any phase
  #
  "ImageCompressor",
  "PdfVisualizer",
  "EdgeDetector",
  "Dehyphenator",
]


[Plugins.Plugin.Init]
path = "@getmorebrain/bitmark-extractor-plugin-init"
Config.imageBaseUri = "https://gmb.image.server.com/replace-me/"

[Plugins.Plugin.PdfExtractPageImages_ForImages]
Config.useCachedData = true
path = "@getmorebrain/bitmark-extractor-plugin-pdf-extract-page-images"
Config.imageScale = 4

[Plugins.Plugin.PdfExtractPageImages_ForVisualization]
Config.useCachedData = true
path = "@getmorebrain/bitmark-extractor-plugin-pdf-extract-page-images"
Config.outputDir = "pdf-images/page-visualization"
Config.imageScale = 2

[Plugins.Plugin.PdfExtractEmbeddedImages]
Config.useCachedData = true

path = "@getmorebrain/bitmark-extractor-plugin-pdf-extract-embedded-images"
[Plugins.Plugin.PdfBlockifier]
Config.useCachedData = true
path = "@getmorebrain/bitmark-extractor-plugin-pdf-blockifier"
Config.Visualization.enabled = true
Config.Visualization.ocrEnabled = true
Config.Visualization.textEnabled = true

[Plugins.Plugin.Runs]
Config.useCachedData = true
path = "@getmorebrain/bitmark-extractor-plugin-runs"
Config.Visualization.enabled = true

[Plugins.Plugin.Tables]
Config.useCachedData = true
path = "@getmorebrain/bitmark-extractor-plugin-tables"
Config.Visualization.enabled = true

[Plugins.Plugin.Images]
Config.useCachedData = true
path = "@getmorebrain/bitmark-extractor-plugin-images"
Config.Visualization.enabled = true

[Plugins.Plugin.Chapters]
Config.useCachedData = true
path = "@getmorebrain/bitmark-extractor-plugin-chapters"
Config.Visualization.enabled = true
Config.invalid = 1

[Plugins.Plugin.Metadata]
Config.useCachedData = true
path = "@getmorebrain/bitmark-extractor-plugin-metadata"
Config.Visualization.enabled = true

[Plugins.Plugin.Paragraphs]
Config.useCachedData = true
path = "@getmorebrain/bitmark-extractor-plugin-paragraphs"
Config.Visualization.enabled = true

[Plugins.Plugin.RegexExample]
Config.useCachedData = false
path = "@getmorebrain/bitmark-extractor-plugin-regex"
Config.Visualization.enabled = true
Config.outputDir = "regex-example"
Config.subBlockJoiner = " "
Config.matches = ['''
  /^([0-9]+[.]{1}[0-9]+)[.]{0,1}[\s]+(.+)$/u
  ''']
Config.blockRole = "chapterHeading"
Config.generateBitmark = true
Config.template = '''
[^.chapter]
[%${1}][%][%${originalPageNo}][%]
[#${2}]
'''
# Config.replacerRules = { 1 = { breakscape = true }, 2 = { breakscape = false } }

[Plugins.Plugin.DocumentMerger]
Config.useCachedData = true
path = "@getmorebrain/bitmark-extractor-plugin-document-merger"
Config.Visualization.enabled = true
Config.Visualization.Unknown.enabled = true
Config.documentsToMerge = [
  "runs.document.json",
  "tables.document.json",
  "images.document.json",
  "metadata-detector.document.json",
  "chapters.document.json",
  "paragraphs.document.json",
  # "regex-example.document.json",
]

[Plugins.Plugin.PageVisualizationGenerator]
Config.useCachedData = true
path = "@getmorebrain/bitmark-extractor-plugin-page-visualization-generator"

[Plugins.Plugin.BitmarkGenerator]
Config.useCachedData = true
path = "@getmorebrain/bitmark-extractor-plugin-bitmark-generator"
Config.Visualization.enabled = true

[Plugins.Plugin.ImageCompressor]
path = "@getmorebrain/bitmark-extractor-plugin-image-compressor"

[Plugins.Plugin.PdfVisualizer]
path = "@getmorebrain/bitmark-extractor-plugin-pdf-visualizer"

[Plugins.Plugin.EdgeDetector]
path = "@getmorebrain/bitmark-extractor-plugin-edge-detector"

[Plugins.Plugin.Dehyphenator]
path = "@getmorebrain/bitmark-extractor-plugin-dehyphenator"

[Plugins.Plugin.HelloWorld]
path = "@getmorebrain/bitmark-extractor-plugin-hello-world"


[General]

# The pdfToImage provider is used to create images of the pages of the document
# Options: mutool, pdftocairo
pdfToImageProvider = "mutool"

# The image data provider is used to extract image metadata from the document
# Options: mutool, pdftohtml
imageDataProvider = "mutool"

# The image data provider is used to extract image metadata from the document
# Options: mutool, pdfimages
imageExtractorProvider = "mutool"


#
# FilterAndExclude configuration
#
[FilterAndExclude]

# Ignore the page / page range
ignorePage = false

# Set the content borders. Areas outside of the border will be ignored
contentBorder.top = 10
contentBorder.left = 10
contentBorder.bottom = 10
contentBorder.right = 10

# Set the excludeBoxes. Areas within these bounding boxes will be ignored
excludeBoxes = []


#
# DetectMetadata configuration
#
[DetectMetadata]

# Set true to enable generation of visualization PDFs.
enableVisualization = false

# Metadata title (if not set, the title will be extracted from the document)
#title = ""

# Metadata subtitle (if not set, the subtitle will be extracted from the document)
#subtitle = ""

# Metadata author (if not set, the author will be extracted from the document)
#author = ""

# Metadata publisher (if not set, the publisher will not be set)
#publisher = ""

# Metadata language (if not set, the language will be extracted from the document)
#language = ""

# Metadata coverImage (if not set, the first page of the document will be used as the cover image)
#coverImage = ""


#
# DetectHeadings configuration
#
[DetectHeadings]

# Set true to enable generation of visualization PDFs.
enableVisualization = false

# The heading exclude regexes to use to exclude headings
headingExcludeRegexes = []


# The levels of headings to detect
[DetectHeadings.headingLevel.1]
enabled = true
heightMin = 0
heightMax = 1000
regex = '''
/^([0-9]+)[.]{0,1}[\s]+(.+)$/u
'''

[DetectHeadings.headingLevel.2]
enabled = true
heightMin = 0
heightMax = 1000
regex = '''
/^([0-9]+[.]{1}[0-9]+)[.]{0,1}[\s]+(.+)$/u
'''

[DetectHeadings.headingLevel.3]
enabled = true
heightMin = 0
heightMax = 1000
regex = '''
/^([0-9]+[.]{1}[0-9]+[.]{1}[0-9]+)[.]{0,1}[\s]+(.+)$/u
'''

[DetectHeadings.headingLevel.4]
enabled = true
heightMin = 0
heightMax = 1000
regex = '''
/^([0-9]+[.]{1}[0-9]+[.]{1}[0-9]+[.]{1}[0-9]+)[.]{0,1}[\s]+(.+)$/u
'''


#
# DetectParagraphs configuration
#
[DetectParagraphs]

# Set true to enable generation of visualization PDFs.
enableVisualization = false

# GMB specific settings
[DetectParagraphs.GetMoreBrain]

[DetectParagraphs.GetMoreBrain.DocumentParagraphs]

maxShortLinePercentage = 0.5
standardIndentPercentage = 0.01
largeIndentPercentage = 0.3
shortLineLengthPercentage = 0.8
veryShortLineLengthPercentage = 0.65
standardWordSpacingPercentage = 0.8
lineHeightAnomonlyPercentage = 0.2

#
# DetectUnknown configuration
#
[DetectUnknown]

# Set true to enable generation of visualization PDFs.
enableVisualization = false


#
# MergeDocuments configuration
#
[MergeDocuments]

# Set true to enable generation of visualization PDFs.
enableVisualization = false

#
# RemoveNewlines configuration
#
[RemoveNewlines]

# The dehyphenator provider is used to dehyphenate words that have been split across lines
# Options: None, Basic, OpenAI
dehyphenationProvider = "Basic"


#
# ExtractImages configuration
#
[ExtractImages]

enableExtractorBlockImages = true

#
# GenerateBitmark configuration
#
[GenerateBitmark]

# Header bit type to be generated.
# Values: none, book (default)
headerBitType = "book"

# Set the base URI for images
imageBaseUri = "https://gmb.image.server.com/replace-me/"

# Scale factor for the sizes of all images
imageScale = 2


[Visualization]

# The scale of the visualization PDFs: 1 = 72 DPI, 2 = 144 DPI, 3 = 216 DPI, 4 = 288 DPI, etc.
scale = 2

#
# Page specific configuration
#
# Use this section to set specific configuration for individual pages, ranges of pages
# NOTE: Pages are indexed from 1
#
# Examples:
# [Pages.1] - Configuration for page 1
# [Pages.5] - Configuration for page 5
# [Pages.7-20] - Configuration for pages 7 to 20
# [Pages.20-] - Configuration for pages 20 to end of document
# [Pages.-200] - Configuration for pages 1-200, equivalent to [Pages.1-200]
#
[Pages]

# Example: setting the content border for a specific page
# [Pages.1.FilterAndExclude]
# contentBorder.top = 20
# contentBorder.left = 20
# contentBorder.bottom = 20
# contentBorder.right = 20

# Example: setting the content border and exclude boxes for a specific page range
# [Pages.4-7.FilterAndExclude]
# contentBorder.top = 25
# contentBorder.left = 10
# contentBorder.bottom = 5
# contentBorder.right = 11

# excludeBoxes = [
#   { x = 16, y = 30, width = 120, height = 220 },
#   { x = 300, y = 400, width = 90, height = 200 },
# ]

# Example: overriding the page range content border for a specific page
# [Pages.5.FilterAndExclude]
# contentBorder.top = 50
# contentBorder.left = 50
# contentBorder.bottom = 50
# contentBorder.right = 50

# Example: ignoring a specific page completely
# [Pages.8.FilterAndExclude]
# ignorePage = true

Eigenschaften

Body-Format
text
Body
Optional
Hat Footer
Ja
Ressourcenanhang
all

bit-identity 11

Name Beschreibung Format Min / Max Standardwert
@id The unique identifier(s) for the bit string 0 - ∞ ""
@gUri Global URI for the bit string 0 - 1 ""
@customerId The customer-specific identifier for the bit string 0 - 1 ""
@externalId The external identifier for the bit string 0 - ∞ ""
▼ Anchor of the bit string 0 - 1 ""
customer▼ Anchor of the bit, as the customer's source names it string 0 - 1 ""
@sourceRL The original location of the information in the original source material string 0 - 1 ""
@sourceBB The source bounding box(es) for the bit (x, y, x1, y1) numberList4 0 - ∞ omit
@bitFingerprint Fingerprint(s) used by the AI extractor to identify the bit (font, style, color, pixels, etc.) string 0 - ∞ ""
@extractorTag The extractor tag(s) for the bit string 0 - ∞ ""
@extractorInternal Internal metadata used by the extractor for processing purposes string 0 - ∞ ""

bit-rights 5

Name Beschreibung Format Min / Max Standardwert
@copyright Copyright of the bit string 0 - 1 ""
@license Licence of the bit string 0 - 1 ""
@rightsPolicy The rights policy or policies for the bit string 0 - ∞ ""
@publisher The publisher(s) of the bit string 0 - ∞ ""
@publisherName The name of the publisher of the bit string 0 - 1 ""

bit-editorial 14

Name Beschreibung Format Min / Max Standardwert
@isTemplate If true, the bit is a template boolean 0 - 1 false
@isTemplateStripTheme If true, the template's theme is stripped boolean 0 - 1 false
@needsValidation If true, the bit needs validation boolean 0 - 1 false
@validationDate The date when the bit was validated string 0 - 1 ""
@aiGenerated If true, the bit is AI-generated boolean 0 - 1 omit
@aiDisclosureRequired If true, readers must be shown that the bit's content is AI-generated boolean 0 - 1 false
@aiModel The AI model used to create the bit's content, e.g. gpt-5, claude-sonnet-4 string 0 - 1 ""
@aiOperation What the AI did to the bit's content, e.g. generate, translate, rewrite, summarize, transform string 0 - 1 ""
@aiReviewed If true, a person has reviewed the bit's AI-generated content as an editor boolean 0 - 1 false
@machineTranslated If true, the bit is machine-translated string 0 - 1 ""
@translationOf Translation source reference string 0 - 1 ""
@reviewTag The review tag(s) for the bit string 0 - ∞ ""
@reviewerTag The reviewer tag(s) for the bit string 0 - ∞ ""
@internalComment Internal comment for the bit string 0 - ∞ ""

bit-display 14

Name Beschreibung Format Min / Max Standardwert
@isCollapsible If true, the bit is collapsible boolean 0 - 1 omit
@collapsed If true, the bit is collapsed boolean 0 - 1 false
@isSelectable If true, the bit is selectable boolean 0 - 1 false
@selected If true, the bit is selected boolean 0 - 1 false
@pageBreak If true, the bit starts on a new page boolean 0 - 1 false
@spansPageBreak If true, the bit spans a page break boolean 0 - 1 omit
@showInIndex If true, the bit is shown in the index boolean 0 - 1 false
@theme The theme(s) for the bit string 0 - ∞ omit
@rtl If true, the book is right-to-left boolean 0 - 1 false
@icon The icon for the bit string 0 - 1 ""
@iconTag The icon tag for the bit string 0 - 1 ""
@colorTag The color tag(s) for the bit string 0 - ∞ ""
@layer The layer(s) for the bit string 0 - ∞ ""
@layerRole The layer role(s) for the bit string 0 - ∞ ""

bit-tagging 11

Name Beschreibung Format Min / Max Standardwert
@searchIndex The search index(es) for the bit string 0 - ∞ ""
@search Search text of the bit string 0 - 1 ""
@tag The tag(s) for the bit string 0 - ∞ omit
@groupTag The group tag(s) for the bit string 0 - ∞ ""
@tag The tag(s) for the group string 0 - ∞ omit
@analyticsTag The analytics tag(s) for the bit string 0 - ∞ ""
@categoryTag The category tag(s) for the bit string 0 - ∞ ""
@topicTag The topic tag(s) for the bit string 0 - ∞ ""
@reportTag The report tag(s) for the bit string 0 - ∞ ""
@reductionTag The reduction tag(s) for the bit string 0 - ∞ ""
@bubbleTag The bubble tag(s) for the bit string 0 - ∞ ""
@target The target(s) for the bit string 0 - ∞ ""

bit-audience 7

Name Beschreibung Format Min / Max Standardwert
@lang The language for the bit string 0 - 1 ""
@altLangTag The alternative language tag for the bit string 0 - 1 ""
@ageRange The age range for the bit string 0 - ∞ ""
@levelCEFRp The CEFRp level for the bit string 0 - 1 ""
@levelCEFR The CEFR level for the bit string 0 - 1 ""
@levelILR The ILR level for the bit string 0 - 1 ""
@levelACTFL The ACTFL level for the bit string 0 - 1 ""

bit-accessibility 2

Name Beschreibung Format Min / Max Standardwert
@accessibilityTag The accessibility classification(s) for the bit enum 0 - ∞ omit
@accessibilityGroupTag Accessibility group tag(s) of the bit string 0 - ∞ ""
@accessibilityTag Accessibility classification(s) of the group enum 0 - ∞ omit

bit-feedback 3

Name Beschreibung Format Min / Max Standardwert
@feedbackEngine The feedback engine for the bit string 0 - 1 ""
@feedbackType The feedback type for the bit string 0 - 1 ""
@disableFeedback If true, feedback is disabled for the bit boolean 0 - 1 false

bit-diff 5

Name Beschreibung Format Min / Max Standardwert
@diffTo The diff to version for the bit string 0 - 1 ""
@diffOp The diff operation for the bit string 0 - 1 ""
@diffRef The diff reference for the bit string 0 - 1 ""
@diffContext The diff context for the bit string 0 - 1 ""
@diffTime The diff time for the bit number 0 - 1 omit

standard-item-lead 1

Name Beschreibung Format Min / Max Standardwert
% The item bitmark++ 0 - 1 ""
% The lead bitmark++ 0 - 1 ""
% The page number bitmark++ 0 - 1 ""
% The margin number bitmark++ 0 - 1 ""

standard-item-lead-instruction-hint 2

Name Beschreibung Format Min / Max Standardwert
! The instruction bitmark++ 0 - 1 ""
? The hint bitmark++ 0 - 1 ""

standard-tags 1

Name Beschreibung Format Min / Max Standardwert
@example The example(s) for the bit bitmark++ 0 - 1 omit