#!/usr/bin/env python

import argparse
import os
import shutil
import subprocess
import tarfile
import zipfile
import html

from collections import OrderedDict
from datetime import datetime, timezone
from typing import List, Optional

from ebooklib import epub
from PIL import Image

import hocr.parse
import hocr.util
import hocr.structure

try:
    from lxml import etree
except ImportError:
    etree = None

try:
    from derivermodule.metadata import parse_item_metadata
    from derivermodule.scandata import (scandata_get_pagetype_pages,
                                        scandata_get_skip_pages,
                                        scandata_parse)
except:
    # This is ok, just don't support _meta.xml and _scandata.xml
    # Just error later on when/if the files are actually being passed
    parse_item_metadata         = None
    scandata_get_skip_pages     = None
    scandata_get_pagetype_pages = None


Image.MAX_IMAGE_PIXELS = 933120000


KAKADU_ERRORS = [
    'Cannot write output with sample precision in excess of 32 bits per sample',
    'Unable to find the compositing layer identified',
]
OCR_SYSTEM_TESSERACT = 'Tesseract'
OCR_SYSTEM_ABBYY = 'ABBYY'
OCR_SYSTEM_UNKNOWN = 'Unknown'

# Running-head furniture detection: lines that repeat at the top of many
# pages (running heads, embedded folios) are dropped from the text, the
# headings and the ToC. See _collect_running_heads for the rules.
FURNITURE_TOP_BAND = 0.12
FURNITURE_MIN_PAGES = 2
FURNITURE_RUN_GAP = 5
FURNITURE_PREFIX_TOKENS = 2

if os.path.exists('/var/tmp/fast'):
    WORKING_DIR = '/var/tmp/fast/'
else:
    WORKING_DIR = '/tmp/'


def image_has_non_fatal_kakadu_errors(
        kakadu_error: str, known_kakadu_errors: List[str]
    ) -> bool:
    """
    Returns True if the Kakadu error should not be fatal to the overall conversion.

    Certain `.jp2` files are corrupt in various ways, and understandably Kakadu
    cannot process them. Indeed, in BookReader they will also not display.
    See, e.g., https://archive.org/details/cu31924003577214/page/n11/mode/2up.

    However, we can salvage the other pages and continue.
    """
    return any(error for error in known_kakadu_errors if error in kakadu_error)


class ImageStack(object):
    filenames = []
    images_per_page = {}
    temp_files = []
    def __init__(self,
                 image_archive_file_path: str,
                 output_basename: str,
                 use_kakadu: bool = False,
                 ignore_broken_images: bool = False):
        self.output_basename = output_basename
        self.image_archive_file_path = image_archive_file_path
        self.is_zip = self.image_archive_file_path.endswith('.zip')
        self.tempdir_zip   = os.path.join(WORKING_DIR, 'kakadu_input')
        self.tempfile_jp2  = os.path.join(WORKING_DIR, 'temp.jp2')
        self.use_kakadu = use_kakadu
        self.ignore_broken_images = ignore_broken_images
        self.parse_stack()

    def parse_stack(self):

        if self.is_zip:
            # Get all the images in the filename order (this should correspond to the page ordering)
            self.zf = zipfile.ZipFile(self.image_archive_file_path)
            for idx, img in enumerate(sorted(self.zf.namelist())):
                info = self.zf.getinfo(img)
                if info.is_dir():
                    continue
                self.filenames.append(img)
        else:
            self.zf = tarfile.open(self.image_archive_file_path)
            for idx, img in enumerate(self.zf):
                if not img.isfile():
                    continue
                self.filenames.append(img.name)
            self.filenames = sorted(self.filenames)

    def crop_image(self, page, box):
        # Keep track of the number of images cropped out from each page
        try:
            self.images_per_page[page] += 1
        except KeyError:
            self.images_per_page[page] = 0

        output_filename = "%s_%04u_%02u.jpeg" % (self.output_basename, page, self.images_per_page[page])
        #return output_filename

        from datetime import datetime
        print("%s - Cropping page %u to box %s" % (datetime.now(), page, box))

        # Extract the image from the zipfile
        tempfile_tiff = os.path.join(WORKING_DIR, 'page_%u.tiff' % page)

        if tempfile_tiff not in self.temp_files:
            extracted_file_path = os.path.join(self.tempdir_zip, self.filenames[page])

            if self.is_zip:
                self.zf.extract(self.filenames[page], self.tempdir_zip)
            else:
                os.makedirs(os.path.dirname(extracted_file_path), exist_ok=True)
                fp = self.zf.extractfile(self.filenames[page])
                f = open(extracted_file_path, 'wb+')
                f.write(fp.read())
                fp.close()
                f.close()

            # TODO: Just use Pillow here, before the rename even
            if self.use_kakadu and self.filenames[page].endswith('.jp2'):
                os.rename(extracted_file_path, self.tempfile_jp2)

                cmd = [
                    'kdu_expand',
                    '-num_threads', str(1),
                    '-i', self.tempfile_jp2,
                    '-o', tempfile_tiff
                ]

                result = subprocess.run(
                    cmd, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True
                )

                if self.ignore_broken_images and result.stderr and image_has_non_fatal_kakadu_errors(result.stderr, KAKADU_ERRORS):
                    print(f'Error in image stack for {extracted_file_path}: {result.stderr}')
                    return
                elif result.returncode != 0:
                    raise RuntimeError(
                        f"Can't convert JP2 to TIFF: {result.stderr}"
                    )

            else:
                i = Image.open(extracted_file_path)
                i.save(tempfile_tiff, compression=None)
                del i
                os.remove(extracted_file_path)

            # Keep track of the temp files so we can delete them later
            self.temp_files.append(tempfile_tiff)

        self.temp_files.append(output_filename)

        #fh = self.zf.open(self.filenames[page])
        img = Image.open(tempfile_tiff)
        if hocr.extutil.needs_grayscale_conversion(image=img):
            img = img.convert('L')
        elif hocr.extutil.needs_rgb_conversion(image=img):
            img = img.convert('RGB')

        #img = Image.open(fh)
        if box:
            region = img.crop(box)
            region.save(output_filename)
        else:
            img.save(output_filename)
        #fh.close()
        img.close()
        return output_filename

    def __del__(self):
        # Close zipfile
        try:
            self.zf.close()
        except:
            pass
        # Clean up temporary files
        try:
            shutil.rmtree(self.tempdir_zip)
            if self.tempfile_jp2 and os.path.exists(self.tempfile_jp2):
                os.unlink(self.tempfile_jp2)
            for tempfile in self.temp_files:
                os.unlink(tempfile)
        except:
            pass


class LangEpubWriter(epub.EpubWriter):
    """EpubWriter that puts ``xml:lang`` on the ``<package>`` element and
    adds a pageList to the NCX (pageTargets) that ebooklib does not build.

    ebooklib hardcodes the package attributes, but validators (Ace's
    ``epub-lang``) want the language on the package element itself, not
    only as a dc:language entry. Older reading systems read only the
    NCX, so the pageTargets carry the same page breaks as nav.xhtml.
    """

    def __init__(self, name, book, options=None, language=None):
        super().__init__(name, book, options)
        self.xml_lang = language
        self.pagebreaks = getattr(book, 'pagebreaks', None) or []

    def _write_opf_file(self, root):
        if self.xml_lang:
            root.set('{http://www.w3.org/XML/1998/namespace}lang',
                     self.xml_lang)
        return super()._write_opf_file(root)

    def _get_nav(self, item):
        tree_str = super()._get_nav(item)
        if etree is not None:
            # Some readers (e.g. Kobo) ignore hidden="hidden" on <nav> and
            # show the nav even when it is not in the spine, so the
            # landmarks nav needs both the hidden attribute and a style
            # rule. The page-list nav already carries hidden; give it the
            # same style so it stays out of the reading flow there too.
            try:
                tree = etree.fromstring(tree_str)
                for nav in tree.iter():
                    nav_type = nav.attrib.get(
                        '{http://www.idpf.org/2007/ops}type')
                    if nav_type in ('landmarks', 'page-list'):
                        nav.set('hidden', 'hidden')
                        nav.set('style', 'display:none')
                tree_str = etree.tostring(
                    tree, xml_declaration=True, encoding='utf-8',
                    pretty_print=True)
            except etree.Error:
                pass
        return tree_str

    def _get_ncx(self):
        tree_str = super()._get_ncx()
        if etree is None or not self.pagebreaks:
            return tree_str
        try:
            tree = etree.fromstring(tree_str)
        except etree.Error:
            return tree_str
        root = tree

        # pageList after the navMap: the same page breaks as nav.xhtml,
        # for reading systems that read only the NCX.
        nav_map = root.find(
            '{http://www.daisy.org/z3986/2005/ncx/}navMap')
        page_list = etree.SubElement(root, 'pageList')
        play_order = 1
        numbers = []
        for fname, leaf, num in self.pagebreaks:
            kind = 'normal' if num.isdigit() else 'front'
            attrib = {'id': 'pt%d' % leaf, 'playOrder': str(play_order),
                      'type': kind}
            if kind == 'normal':
                numbers.append(int(num))
                attrib['value'] = str(num)
            play_order += 1
            page_target = etree.SubElement(page_list, 'pageTarget', attrib)
            nav_label = etree.SubElement(page_target, 'navLabel')
            text = etree.SubElement(nav_label, 'text')
            text.text = num
            etree.SubElement(page_target, 'content', {
                'src': '%s#pg%d' % (fname, leaf)})

        # dtb:totalPageCount / dtb:maxPageNumber from the arabic-numbered
        # pages; ebooklib hardcodes both to 0.
        head = root.find(
            '{http://www.daisy.org/z3986/2005/ncx/}head')
        for meta in head.findall(
                '{http://www.daisy.org/z3986/2005/ncx/}meta'):
            name = meta.get('name')
            if name == 'dtb:totalPageCount':
                meta.set('content', str(len(numbers)))
            elif name == 'dtb:maxPageNumber':
                meta.set('content', str(max(numbers) if numbers else 0))

        return etree.tostring(tree, xml_declaration=True,
                              encoding='utf-8', pretty_print=True)


def write_epub_file(path, book, language=None):
    """Write the EPUB with :class:`LangEpubWriter` (xml:lang support)."""
    writer = LangEpubWriter(path, book, {}, language=language)
    writer.process()
    try:
        writer.write()
    except IOError:
        pass


class PageHtml(epub.EpubHtml):
    """A page that can omit the ``lang``/``xml:lang`` attributes.

    ebooklib always writes ``lang`` from ``self.lang or self.book.language`` and
    lxml cannot store ``None``. When no language is known we set a sentinel value
    so the base class can serialize, then strip it back out of the result.
    """
    omit_lang = False

    def get_content(self, default=None):
        if not self.omit_lang:
            return super().get_content(default)

        sentinel = '__OMIT_LANG__'
        original_lang = self.lang
        self.lang = sentinel
        try:
            content = super().get_content(default)
        finally:
            self.lang = original_lang

        content = content.replace((' lang="%s"' % sentinel).encode('utf-8'), b'')
        content = content.replace((' xml:lang="%s"' % sentinel).encode('utf-8'), b'')
        return content


class EpubGenerator(object):

    __version__ = '1.0.0'

    front_matter = (
        '<div class="offset">'
        '<p dir="ltr">This book was produced in EPUB format by the '
        'Internet Archive.</p> '
        '<p dir="ltr">The book pages were scanned and converted to EPUB '
        'format automatically. This process relies on optical character '
        'recognition, and is somewhat susceptible to errors. The book may '
        'not offer the correct reading sequence, and there may be '
        'weird characters, non-words, and incorrect guesses at '
        'structure. Some page numbers and headers or footers may remain '
        'from the scanned page. The process which identifies images might '
        'have found stray marks on the page which are not actually images '
        'from the book. The hidden page numbering which may be available '
        'to your ereader corresponds to the numbered pages in the print '
        'edition, but is not an exact match;  page numbers will increment '
        'at the same rate as the corresponding print edition, but we may '
        'have started numbering before the print book\'s visible page '
        'numbers.  The Internet Archive is working to improve the '
        'scanning process and resulting books, but in the meantime, we '
        'hope that this book will be useful to you.</p> '
        '<p dir="ltr">The Internet Archive was founded in 1996 to build '
        'an Internet library and to promote universal access to all '
        'knowledge. The Archive\'s purposes include offering permanent '
        'access for researchers, historians, scholars, people with '
        'disabilities, and ' 'the general public to historical '
        'collections that exist in digital format. The Internet Archive '
        'includes texts, audio, moving images, '
        'and software as well as archived web pages, and provides '
        'specialized services for information access for the blind and '
        'other persons with disabilities.</p>'
        '<p>Created with hocr-to-epub (v.%s)</p></div>'
    ) % __version__

    # define CSS style
    style = """
        .center {text-align: center}
        .sr-only {
            width: 1px;
            height: 1px;
            padding: 0;
            margin: -1px;
            overflow: hidden;
            clip: rect(0,0,0,0);
            border: 0;
        }
        .strong {font-weight: bold;}
        .italic {font-style: italic;}
        .serif {font-family: serif;}
        .sans {font-family: sans-serif;}
        .big {font-size: 1.5em;}
        .small {font-size: .75em;}
        .offset {
            margin: 1em;
            padding: 1.5em;
            border: black 1px solid;
        }
        img {
            padding: 0;
            margin: 0;
            max-width: 100%;
            max-height: 100%;
            column-count: 1;
            break-inside: avoid;
            oeb-column-number: 1;
        }
        /* Paragraphs are indented and not separated by a blank line, which
           is how a book is set and how a reader's other books look.
           Leaving <p> unstyled is not neutral: the reading system falls
           back to its own default of margin:1em and no indent, so the
           book arrives looking like a web page. Follows Standard Ebooks'
           core.css.

           No accessibility standard requires either convention -- WCAG
           1.4.12 asks only that the text survive a reader overriding the
           spacing, which this does, since the override adds margin we
           have not spent. */
        p {
            margin: 0;
            text-indent: 1em;
        }
        /* The opening paragraph of anything is not indented: there is
           nothing above it to be distinguished from. */
        h1 + p, h2 + p, hr + p {
            text-indent: 0;
        }
        /* The printed page number, shown where that page began. Colour
           is inherited rather than set, so it follows the reader's own
           theme and keeps whatever contrast they have chosen. */
        .pagenum {
            font-size: 0.75em;
            vertical-align: super;
        }
        """""

    # Sentence terminators that end a paragraph; anything else and a
    # lowercase opening on the next paragraph means the two are one
    # sentence split by a page turn or an OCR paragraph break.
    SENTENCE_ENDINGS = '.!?\u201d\u2019\u2026:;)'

    strip_whitespaces = True

    def __init__(self,
                 hocr_xml_file_path: str,
                 meta_xml_file_path: Optional[str] = None,
                 image_stack_zip_file_path: Optional[str] = None,
                 scandata_xml_file_path: Optional[str] = None,
                 epub_zip_file_path: Optional[str] = None,
                 use_kakadu: bool = False,
                 ignore_broken_images: bool = False,
                 image_kinds: Optional[set] = None):

        # Copy arguments to locals
        self.hocr_xml_file_path = hocr_xml_file_path
        self.meta_xml_file_path = meta_xml_file_path
        self.image_stack_zip_file_path = image_stack_zip_file_path
        self.scandata_xml_file_path = scandata_xml_file_path
        self.epub_zip_file_path = epub_zip_file_path
        self.image_kinds = set(image_kinds or [])

        # Set sensible defaults for arguments that weren't provided
        if not self.meta_xml_file_path:
            self.meta_xml_file_path = self.hocr_xml_file_path.replace('_hocr.html', '_meta.xml')
        if not self.image_stack_zip_file_path:
            self.image_stack_zip_file_path = self.hocr_xml_file_path.replace('_hocr.html', '_jp2.zip')
        if not self.scandata_xml_file_path:
            self.scandata_xml_file_path = self.hocr_xml_file_path.replace('_hocr.html', '_scandata.xml')
        if not self.epub_zip_file_path:
            self.epub_zip_file_path = self.hocr_xml_file_path.replace('_hocr.html', '_ebook.epub')

        # We can still make an epub without any of these if we must
        # Try to find metadata
        if os.path.exists(self.meta_xml_file_path) and parse_item_metadata is not None:
            self.metadata = parse_item_metadata(self.meta_xml_file_path)
        else:
            self.metadata = {}
        # Try to find jp2.zip
        if os.path.exists(self.image_stack_zip_file_path):
            self.img_stack = ImageStack(
                    self.image_stack_zip_file_path,
                    os.path.join(WORKING_DIR,"epub_img"),
                    use_kakadu=use_kakadu,
                    ignore_broken_images=ignore_broken_images)
        else:
            self.img_stack = None
        # Try to find scandata
        if os.path.exists(self.scandata_xml_file_path) \
        and scandata_get_skip_pages is not None \
        and scandata_get_pagetype_pages is not None:
            self.scandata    = scandata_parse(self.scandata_xml_file_path)
            self.skip_pages  = scandata_get_skip_pages(self.scandata)
            self.cover_pages = scandata_get_pagetype_pages(self.scandata, 'Cover')

            # Ignore all cover pages if one cover page is detected as cloth
            # cover (in this case the other cover pages are likely also cloth
            # covers)
            sd_pages = self.scandata['book']['pageData']['page']
            if not isinstance(sd_pages, list):
                sd_pages = [sd_pages]
            for cover_page in self.cover_pages:
                if sd_pages[cover_page].get('clothCover', None) == 'true':
                    self.cover_pages = []
                    break

            # Map leaf -> printed page number from the scandata page entries,
            # skipping leaves excluded from access formats. Used as-is (no
            # interpolation), then deduplicated against the pageNumData
            # assertions below.
            self.page_numbers = {}
            self.asserted_leaf_numbers = {}
            for p in sd_pages:
                if str(p.get('addToAccessFormats', 'true')).lower() == 'false':
                    continue
                num = str(p.get('pageNumber') or '').strip()
                if num:
                    self.page_numbers[int(p['@leafNum'])] = num
            # pageNumData assertions: the page-number tool's confirmed
            # anchors. They win over the interpolated page entries.
            asrt = self.scandata.get('book', {}).get('pageNumData', {})
            if isinstance(asrt, dict):
                for assertion in asrt.get('assertion') or []:
                    if not isinstance(assertion, dict):
                        continue
                    leaf = assertion.get('leafNum')
                    num = (assertion.get('pageNum') or '').strip()
                    if leaf is None or not num:
                        continue
                    self.asserted_leaf_numbers[int(leaf)] = num
            self.page_numbers = self._reconcile_duplicate_page_numbers(
                self.page_numbers)
        else:
            self.skip_pages = []
            self.cover_pages = []
            self.page_numbers = {}

        print("Parsing file %s" % self.hocr_xml_file_path)
        self.generate()

    def _reconcile_duplicate_page_numbers(self, page_numbers):
        """Resolve duplicated printed numbers to a single leaf.

        The page-number tool interpolates across unnumbered plates, so a
        number can land on two leaves and collide with the next anchor.
        A reader that follows the page-list would jump to an unnumbered
        plate. Prefer the leaf the page-number tool asserted (the
        confirmed anchor); otherwise the earliest leaf carrying the
        number, which is where the folio was printed.
        """
        by_number = {}
        for leaf, num in page_numbers.items():
            by_number.setdefault(num, []).append(leaf)
        for num, leaves in by_number.items():
            if len(leaves) < 2:
                continue
            asserted = [leaf for leaf in leaves
                        if self.asserted_leaf_numbers.get(leaf) == num]
            keep = asserted[0] if asserted else max(leaves)
            for leaf in leaves:
                if leaf != keep:
                    page_numbers.pop(leaf, None)
        return page_numbers

    def _ocr_detected_language_confident(self, threshold=0.9):
        """True if a confident OCR-detected language is available in the metadata."""
        if not self.metadata.get('ocr_detected_lang'):
            return False
        try:
            return float(self.metadata.get('ocr_detected_lang_conf')) >= threshold
        except (TypeError, ValueError):
            return False

    def set_metadata(self):
        """
        Set the metadata on the epub object
        """
        if 'identifier' in self.metadata.keys():
            self.book.set_identifier(self.metadata['identifier'])
        # Determine the book language(s). Prefer explicit metadata; fall back to the
        # OCR-detected language when it is reported confidently.
        raw_lang = self.metadata.get('language')
        if raw_lang:
            raw_languages = raw_lang if isinstance(raw_lang, list) else [raw_lang]
        elif self._ocr_detected_language_confident():
            raw_languages = [self.metadata['ocr_detected_lang']]
        else:
            raw_languages = []

        languages = [
            normalized for normalized in
            (hocr.extutil.normalize_language(lang) for lang in raw_languages)
            if normalized
        ]

        if languages:
            # The HTML lang attribute holds a single tag, so use the first language
            # as the primary and emit every language as its own dc:language entry
            # rather than joining them into one invalid tag.
            self.language = languages[0]
            self.book.set_language(self.language)
            for extra_language in languages[1:]:
                self.book.add_metadata('DC', 'language', extra_language)
        else:
            self.language = None
        if 'title' in self.metadata.keys():
            if isinstance(self.metadata['title'], str):
                self.book.set_title(self.metadata['title'])
            elif isinstance(self.metadata['title'], list):
                title = ' ; '.join(self.metadata['title'])
                self.book.set_title(title)
        if 'creator' in self.metadata.keys():
            if type(self.metadata['creator']) is str:
                self.book.add_author(self.metadata['creator'])
            elif type(self.metadata['creator']) is list:
                for i, creator in enumerate(self.metadata['creator']):
                    creator_uid = 'creator_{creator_uid}'.format(creator_uid=i)
                    self.book.add_author(creator, uid=creator_uid)
        if 'description' in self.metadata.keys():
            if type(self.metadata['description']) is str:
                self.book.add_metadata('DC', 'description', self.metadata['description'])
            elif type(self.metadata['description']) is list:
                for description in self.metadata['description']:
                    self.book.add_metadata('DC', 'description', description)
        if 'publisher' in self.metadata.keys():
            if type(self.metadata['publisher']) is str:
                    self.book.add_metadata('DC', 'publisher', self.metadata['publisher'])
            elif type(self.metadata['publisher']) is list:
                for publisher in self.metadata['publisher']:
                    self.book.add_metadata('DC', 'publisher', publisher)
        if 'identifier-access' in self.metadata.keys():
            if type(self.metadata['identifier-access']) is str:
                self.book.add_metadata(
                    'DC', 'identifier', 'Access URL: {}'.format(
                        self.metadata['identifier-access']
                    )
                )
            elif type(self.metadata['identifier-access']) is list:
                for identifier_access in self.metadata['identifier-access']:
                    self.book.add_metadata(
                        'DC', 'identifier', 'Access URL: {}'.format(
                            identifier_access
                        )
                    )
        if 'identifier-ark' in self.metadata.keys():
            if type(self.metadata['identifier-ark']) is str:
                self.book.add_metadata(
                    'DC', 'identifier', 'urn:ark:{}'.format(self.metadata['identifier-ark'])
                )
            elif type(self.metadata['identifier-ark']) is list:
                for identifier_ark in self.metadata['identifier-ark']:
                    self.book.add_metadata(
                        'DC', 'identifier', 'urn:ark:{}'.format(identifier_ark)
                    )
        if 'isbn' in self.metadata.keys():
            if type(self.metadata['isbn']) is str:
                self.book.add_metadata(
                    'DC', 'identifier', 'urn:isbn:{}'.format(self.metadata['isbn'])
                )
            elif type(self.metadata['isbn']) is list:
                for isbn in self.metadata['isbn']:
                    self.book.add_metadata(
                        'DC', 'identifier', 'urn:isbn:{}'.format(isbn)
                    )
        if 'oclc-id' in self.metadata.keys():
            if type(self.metadata['oclc-id']) is str:
                self.book.add_metadata(
                    'DC', 'identifier', 'urn:oclc:{}'.format(self.metadata['oclc-id'])
                )
            elif type(self.metadata['oclc-id']) is list:
                for oclc_id in self.metadata['oclc-id']:
                    self.book.add_metadata(
                        'DC', 'identifier', 'urn:oclc:{}'.format(oclc_id)
                    )
        if 'external-identifier' in self.metadata.keys():
            if type(self.metadata['external-identifier']) is str:
                self.book.add_metadata('DC', 'identifier', self.metadata['external-identifier'])
            elif type(self.metadata['external-identifier']) is list:
                for external_identifier in self.metadata['external-identifier']:
                    self.book.add_metadata('DC', 'identifier', external_identifier)
        if 'related-external-id' in self.metadata.keys():
            if type(self.metadata['related-external-id']) is str:
                self.book.add_metadata('DC', 'identifier', self.metadata['related-external-id'])
            elif type(self.metadata['related-external-id']) is list:
                for related_external_id in self.metadata['related-external-id']:
                    self.book.add_metadata('DC', 'identifier', related_external_id)
        if 'subject' in self.metadata.keys():
            if type(self.metadata['subject']) is str:
                self.book.add_metadata('DC', 'subject', self.metadata['subject'])
            elif type(self.metadata['subject']) is list:
                for subject in self.metadata['subject']:
                    self.book.add_metadata('DC', 'subject', subject)
        if 'date' in self.metadata.keys():
            if isinstance(self.metadata['date'], str):
                self.book.add_metadata('DC', 'date', self.metadata['date'])
            elif isinstance(self.metadata['date'], list):
                for date in self.metadata['date']:
                    self.book.add_metadata('DC', 'date', date)

    def set_accessibility_metadata(self):
        summary = ''
        # Add the accessibility metadata to the publication
        summary += (
            'The publication was generated using automated character '
            'recognition, therefore it may not be an accurate rendition '
            'of the original text, and it may not offer the correct '
            'reading sequence.'
        )

        modes = []
        modes_sufficient = []
        if self.has_text:
            modes.append('textual')
            modes_sufficient.append('textual')
        if self.has_images:
            modes.append('visual')
            modes_sufficient.append('visual')
            summary += ' This publication is missing meaningful alternative text.'
        if self.low_confidence_pages or self.skipped_low_confidence_pages:
            summary += (' Pages with poor OCR quality are listed in the '
                        'notice at the beginning of the book.')

        self.book.add_metadata(
            None,
            'meta',
            summary,
            OrderedDict([('property', 'schema:accessibilitySummary')])
        )

        for mode in modes:
            self.book.add_metadata(
                None,
                'meta',
                mode,
                OrderedDict([('property', 'schema:accessMode')])
            )

        for mode in modes_sufficient:
            self.book.add_metadata(
                None,
                'meta',
                mode,
                OrderedDict([('property', 'schema:accessModeSufficient')])
            )

        # Derive the accessibility features from what this build actually emitted
        # instead of asserting a fixed value. Add a feature here only when the
        # corresponding affordance is present in the output (a real ToC, page-list
        # markers, meaningful alt text, headings...). Fall back to "none" only
        # when nothing applies.
        features = []
        if self.has_text:
            features.append('displayTransformability')
        if self.toc_entries:
            features.append('tableOfContents')
            features.append('structuralNavigation')
        if self.page_numbers:
            # Markers in the content and the page-list that navigates them are
            # deliberately two separate features (split in 2023; the
            # deprecated printPageNumbers conflated them).
            features.append('pageBreakMarkers')
            features.append('pageNavigation')
        if not features:
            features = ['none']

        for feature in features:
            self.book.add_metadata(
                None,
                'meta',
                feature,
                OrderedDict([('property', 'schema:accessibilityFeature')])
            )

        # these states will be true for any static content,  which we know
        # is guaranteed for OCR generated texts.
        hazards = [
            'noFlashingHazard',
            'noMotionSimulationHazard',
            'noSoundHazard',
        ]
        controls = [
            'fullKeyboardControl',
            'fullMouseControl',
            'fullSwitchControl',
            'fullTouchControl',
            'fullVoiceControl',
        ]

        for hazard in hazards:
            self.book.add_metadata(
                None,
                'meta',
                hazard,
                OrderedDict([('property', 'schema:accessibilityHazard')])
            )
        for control in controls:
            self.book.add_metadata(
                None,
                'meta',
                control,
                OrderedDict([('property', 'schema:accessibilityControl')])
            )


    def _compute_body_size(self, sample_pages=200):
        """Modal word font size across a sample of pages (the body text size).

        Sampled rather than whole-book so it does not need a second full
        parse; the body size is stable across a book.
        """
        sizes = []
        for i, page in enumerate(
                hocr.parse.hocr_page_iterator(self.hocr_xml_file_path)):
            if i >= sample_pages:
                break
            for element in hocr.parse.hocr_page_to_word_data(page):
                for line in element['lines']:
                    for word in line['words']:
                        if word['fontsize']:
                            sizes.append(word['fontsize'])
        return hocr.structure.body_size(sizes)

    def _pagebreak_marker(self, leaf):
        """EPUB pagebreak marker for this leaf's printed page number, or None.

        The marker is what ebooklib's page-list scans for. The number is
        a text node in the document (DAISY's Pipeline reads text, not
        aria-label), with ``aria-label`` kept for screen readers whose
        doc-pagebreak implementation hides the text from assistive tech
        (DAISY's own guidance: aria-label is "currently required even
        when the number is visible"). The class says which sequence the
        number belongs to: ``page-normal`` for arabic body folios,
        ``page-front`` for roman front-matter folios. There is one
        marker per page file, so the leaf number is a safe unique anchor.
        """
        num = self.page_numbers.get(leaf)
        if not num:
            return None
        cls = 'page-normal' if num.isdigit() else 'page-front'
        return ('<span xmlns:epub="http://www.idpf.org/2007/ops" '
                'epub:type="pagebreak" role="doc-pagebreak" '
                'aria-label="%s" class="pagenum %s" id="pg%d">%s</span>'
                % (html.escape(num), cls, leaf, html.escape(num)))

    def _add_page_break_source(self):
        """Emit pageBreakSource identifying the scanned edition.

        Validators require it whenever a reflowable EPUB ships an authored
        page-list, and it is useful provenance regardless: it says which
        printed edition the page numbers refer to.
        """
        if not self.page_numbers:
            return

        def first(value):
            if isinstance(value, list):
                return value[0] if value else None
            return value

        url = first(self.metadata.get('identifier-access'))
        if not url:
            ident = first(self.metadata.get('identifier'))
            if ident:
                url = 'https://archive.org/details/%s' % ident
        parts = [p for p in (first(self.metadata.get('title')),
                            first(self.metadata.get('publisher')),
                            first(self.metadata.get('date'))) if p]
        citation = '. '.join(parts)
        if url:
            citation = (citation + '. ' + url) if citation else url
        if citation:
            self.book.add_metadata(
                None, 'meta', citation,
                OrderedDict([('property', 'pageBreakSource')])
            )

    def _clean_heading_title(self, text):
        """Clean a heading for use as a ToC/landmark title.

        Strips a trailing lowercase roman folio ("PREFACE. xi" ->
        "PREFACE") and stray punctuation ("Dedication ." ->
        "Dedication"), while keeping chapter identifiers that are part of
        the title itself ("CHAPTER XL" stays "CHAPTER XL").
        """
        text = (text or '').strip()
        tokens = text.split()
        while len(tokens) > 1:
            last = tokens[-1].rstrip('.,;:')
            if last and hocr.structure.is_roman_numeral(last) \
                    and last.islower():
                tokens.pop()
                continue
            break
        return ' '.join(tokens).rstrip(' .,;:-').strip()

    def _page_reference(self, page_idx):
        """How to refer to a page in the notice: the printed page number
        when the scan recorded one, otherwise the scanned page (leaf)
        number. Returns ``(label, is_printed)``.
        """
        printed = self.page_numbers.get(page_idx)
        if printed:
            return str(printed), True
        return str(page_idx), False

    def _confidence_notice(self, total_pages):
        """Notice-page paragraph(s) about low-confidence pages, or ''.

        The per-page "only X% accurate" warning used to be prepended to
        every affected page, announcing a number mid-reading that the
        reader cannot act on and cannot dismiss. It now lives in the
        notice the reader sees once at the start: a short list of affected
        pages when only a few are affected, otherwise a single honest
        sentence about the book as a whole.
        """
        paragraphs = []
        if self.low_confidence_pages:
            count = len(self.low_confidence_pages)
            # Listing every page is only useful when it stays short.
            if count <= 25 and count <= 0.10 * max(total_pages, 1):
                printed = []
                scanned = []
                for page_idx, _confidence in self.low_confidence_pages:
                    label, is_printed = self._page_reference(page_idx)
                    (printed if is_printed else scanned).append(label)

                def group(labels, singular, plural):
                    if not labels:
                        return None
                    if len(labels) == 1:
                        return '%s %s' % (singular, labels[0])
                    return '%s %s' % (plural, ', '.join(labels))

                where = [group(printed, 'printed page', 'printed pages'),
                         group(scanned, 'scan page', 'scan pages')]
                paragraphs.append(
                    '<p>Some pages of this book were scanned at poor '
                    'quality: %s. The text on these pages may contain '
                    'many errors.</p>' % ' and '.join(
                        part for part in where if part))
            else:
                if count >= total_pages:
                    extent = 'all %d scanned pages' % total_pages
                else:
                    extent = ('%d of %d scanned pages'
                              % (count, total_pages))
                paragraphs.append(
                    '<p>The OCR quality is poor throughout this book '
                    '(%s); the text may contain many errors.</p>' % extent)
        if self.skipped_low_confidence_pages:
            skipped = ', '.join(
                'scan page %d' % page_idx for page_idx,
                _confidence in self.skipped_low_confidence_pages)
            paragraphs.append(
                '<p>The following pages could not be read at all and are '
                'missing from this EPUB: %s.</p>' % skipped)
        return ''.join(paragraphs)

    def _is_furniture_line(self, page_idx, line, page_height):
        """True if this line was flagged as running-head furniture.

        Recomputes the same normalized key the detection pass used, so the
        decision is deterministic; lines outside the top band are never
        dropped.
        """
        if not self.furniture_drop:
            return False
        if page_height and line['bbox'][1] > FURNITURE_TOP_BAND * page_height:
            return False
        raw = ' '.join(w['text'] for w in line['words']).strip()
        if not raw:
            return False
        folio = self.page_numbers.get(page_idx, '')
        _numeral, rest = hocr.structure.split_folio_token(raw, folio)
        norm = hocr.structure.normalize_head_text(rest)
        return (page_idx, norm) in self.furniture_drop

    def _collect_running_heads(self, body_size):
        """Cross-page pass: find lines that repeat at the top of many pages.

        Running heads ("PRIDE AND PREJUDICE.", "THE BOOK OF LOVE 107",
        "PREFACE. xi") are not content: they land in the prose as stray
        lines, get promoted to headings and then pollute the ToC. Font size
        alone cannot separate them from real section titles — The Book of
        Love sets both at the same size — so this pass uses position and
        recurrence:

        * a line in the top band of the page whose normalized text recurs
          on enough pages joins a "furniture family" (a pair of lines only
          counts when they read like heads, i.e. are mostly caps, so
          recurring body text is safe);
        * occurrences whose text extends a family phrase by a couple of
          tokens are mangled variants of the same head ("THE BOOK OF LOVE
          of" for "THE BOOK OF LOVE 107") and join the family;
        * within a family, occurrences that sit more than
          FURNITURE_RUN_GAP pages apart are separate runs — a chapter
          heading that recurs as a running head keeps its first occurrence
          (the genuine heading) of every run;
        * an occurrence is kept if it is set at body size or larger, or if
          it is the first heading-classifying occurrence of its run and the
          family is not folio-styled (running heads that embed the page's
          own printed number are never kept).

        Returns the set of ``(page index, normalized text)`` to drop.
        """
        occurrences = []
        for page_idx, page in enumerate(
                hocr.parse.hocr_page_iterator(self.hocr_xml_file_path)):
            if page_idx in self.skip_pages:
                continue
            try:
                _width, height = hocr.parse.hocr_page_get_dimensions(page)
            except Exception:
                continue
            if not height:
                continue
            limit = FURNITURE_TOP_BAND * height
            for element in hocr.parse.hocr_page_to_word_data(page):
                for line in element['lines']:
                    if not line['words'] or line['bbox'][1] > limit:
                        continue
                    raw = ' '.join(w['text'] for w in line['words']).strip()
                    if not raw:
                        continue
                    fontsize = max((w['fontsize'] or 0)
                                   for w in line['words'])
                    folio = self.page_numbers.get(page_idx, '')
                    _numeral, rest = hocr.structure.split_folio_token(
                        raw, folio)
                    norm = hocr.structure.normalize_head_text(rest)
                    if not norm and _numeral is None:
                        # Nothing left after stripping: an ornament line
                        # ("—", "***") that carries no words at all. Not
                        # a head, and not worth a family.
                        continue
                    occurrences.append({'page': page_idx, 'raw': raw,
                                        'fontsize': fontsize, 'norm': norm})
        return self._decide_furniture(occurrences, body_size)

    def _decide_furniture(self, occurrences, body_size):
        """Apply the furniture rules to collected top-band occurrences.

        See :meth:`_collect_running_heads`. Returns the set of
        ``(page index, normalized text)`` pairs to drop.
        """
        by_norm = {}
        for occurrence in occurrences:
            occurrence.setdefault('orig_norm', occurrence['norm'])
            by_norm.setdefault(occurrence['norm'], []).append(occurrence)

        drop = set()
        # Folio-only lines (nothing left after stripping the printed
        # number): the pagebreak marker carries that number, so the line
        # is furniture. This runs before the family logic because a
        # folio-only line recurs on every page and would otherwise form
        # a family whose first occurrence survives as a heading.
        folio_only = {o['page'] for norm, occs in by_norm.items()
                      if not norm for o in occs}
        for page in sorted(folio_only):
            drop.add((page, ''))

        # A real running head recurs a lot; a pair of identical lines is
        # only trusted when the lines read like heads (mostly caps), so
        # recurring body text (dialogue, letters) is safe.
        families = set()
        for norm, occs in by_norm.items():
            if not norm:
                continue
            if len(occs) >= 3 or (
                    len(occs) >= FURNITURE_MIN_PAGES
                    and all(hocr.structure.is_mostly_caps(o['raw'])
                            for o in occs)):
                families.add(norm)
        if not families:
            # Only folio-only lines were collected: return them without
            # running the family logic at all.
            return drop

        # Mangled variants of a head ("THE BOOK OF LOVE of" for
        # "THE BOOK OF LOVE 107") join the family of the phrase they
        # extend, as long as the extension is short.
        family_norms = sorted(families, key=len, reverse=True)
        for occurrence in occurrences:
            if occurrence['norm'] in families:
                continue
            for family_norm in family_norms:
                if occurrence['norm'].startswith(family_norm + ' ') and \
                        len(occurrence['norm'].split()) - \
                        len(family_norm.split()) <= FURNITURE_PREFIX_TOKENS:
                    occurrence['norm'] = family_norm
                    by_norm[family_norm].append(occurrence)
                    break

        drop = set()
        for norm, occs in by_norm.items():
            if norm not in families:
                continue
            occs = sorted(occs, key=lambda o: o['page'])
            # Occurrences more than FURNITURE_RUN_GAP pages apart belong to
            # separate runs; each run keeps its first genuine heading.
            runs = [[occs[0]]]
            for prev, cur in zip(occs, occs[1:]):
                if cur['page'] - prev['page'] > FURNITURE_RUN_GAP:
                    runs.append([cur])
                else:
                    runs[-1].append(cur)
            median_size = sorted(o['fontsize'] for o in occs)[len(occs) // 2]
            # A family whose occurrences carry the page's own printed folio
            # at the line edge is a running-head family ("THE BOOK OF LOVE
            # 9", "29 PRIDE AND PREJUDICE."), so its lines are never kept
            # as headings even when they classify like one.
            folio_styled = sum(
                1 for o in occs
                if hocr.structure.carries_folio(
                    o['raw'], self.page_numbers.get(o['page']))) >= 2
            for run in runs:
                first_heading = None
                for occurrence in run:
                    size = occurrence['fontsize'] or median_size
                    if size >= body_size:
                        continue  # body-sized or larger: real text
                    if not folio_styled and first_heading is None and \
                            hocr.structure.classify(
                                occurrence['raw'], occurrence['fontsize'],
                                body_size)[0] == 'heading':
                        first_heading = occurrence
                        continue
                    drop.add((occurrence['page'],
                              occurrence['orig_norm']))
        return drop

    def _continues_sentence(self, first_text, second_text):
        """True if two paragraph texts form one sentence split by a page
        turn or an OCR paragraph break.

        A paragraph continues the sentence when it ends without a
        sentence terminator and the next starts lowercase. A hyphenated
        word split across the break (marked by the DEL placeholder) is a
        continuation too, but only when the continuation itself starts
        lowercase: the other half of the word does.
        """
        first = (first_text or '').rstrip()
        second = (second_text or '').lstrip()
        if not first or not second:
            return False
        if first.endswith('\x7f'):
            return bool(second[:1].islower())
        if first[-1] in self.SENTENCE_ENDINGS:
            return False
        if not second[:1].islower():
            return False
        return True

    def _join_paragraphs(self, items):
        """Merge paragraph items that form one sentence split by a page
        turn or an OCR paragraph break.

        The pagebreak markers that sat between the two halves move to the
        join point inside the merged paragraph, so the page-list still
        points at the right place. A heading or horizontal rule between
        the two halves interrupts the sentence and stops the joining.
        """
        out = []
        i = 0
        n = len(items)
        while i < n:
            item = items[i]
            if item['kind'] != 'para':
                out.append(item)
                i += 1
                continue
            text = item['text']
            j = i + 1
            pending = []
            while j < n:
                nxt = items[j]
                if nxt['kind'] == 'marker':
                    pending.append(nxt)
                    j += 1
                    continue
                if nxt['kind'] != 'para':
                    break       # a heading or hr interrupts the sentence
                if not self._continues_sentence(text, nxt['text']):
                    break
                join = ''.join(marker['html'] for marker in pending)
                pending = []
                if text.endswith('\x7f'):
                    text = text[:-1].rstrip() + join + \
                        nxt['text'].lstrip()
                else:
                    text = text.rstrip() + ' ' + join + nxt['text']
                j += 1
            # Flush markers that could not be merged (the next item was
            # not a continuing paragraph) back into the output order.
            out.extend(pending)
            if text.endswith('\x7f'):
                # A trailing placeholder marks a hyphenated word split
                # across the break; without a lowercase continuation the
                # word is complete, so the placeholder goes away.
                text = text[:-1].rstrip()
            out.append({'kind': 'para', 'page': item['page'],
                        'top_y': item['top_y'], 'text': text})
            i = j
        return out

    def _render_items(self, items):
        """Render a flat list of content items into page HTML."""
        parts = []
        for item in items:
            kind = item['kind']
            if kind == 'heading':
                level = item['level']
                parts.append(u"<h%d>%s</h%d>" % (level, item['text'], level))
            elif kind == 'para':
                parts.append(u"<p>%s</p>" % item['text'])
            elif kind == 'hr':
                parts.append('<hr/>')
            elif kind == 'image':
                parts.append(u"<img src=\"%s\" alt=\"\"/>" % item['src'])
            elif kind == 'marker':
                parts.append(item['html'])
        return ''.join(parts)

    def _section_kind(self, title):
        """Which section a level-1 heading starts: a chapter, a front-matter
        section, or None (a display line that belongs to the current
        section).
        """
        if hocr.structure.is_chapter(title):
            return 'chapter'
        if hocr.structure.front_matter_type(title):
            return 'front'
        # All-caps body sections with a leading article ("THE BOOK OF
        # DANIEL") are chapter starts too. A single-word junk heading
        # (the half-title's "BOOK") is not.
        if hocr.structure.is_caps_heading(title) and \
                hocr.structure.CHAPTER_THE_RE.match(title):
            return 'chapter'
        return None

    def _group_sections(self, page_data):
        """Group the pages' content items into sections, one XHTML file
        each.

        A level-1 chapter heading starts a chapter; a level-1 front-matter
        heading (preface, contents, list of illustrations...) starts a
        front-matter section; content before the first heading forms the
        preamble. Sections can start mid-page, since a chapter opening
        may follow the previous chapter's text on the same page. This is
        what lets paragraphs continue across page turns without being
        split by the file boundary.
        """
        sections = []
        current = None

        def new_section(kind, title):
            return {'kind': kind, 'title': title, 'items': [],
                    'file': None, 'headings': []}

        for data in page_data:
            # A page that lists several chapters is the printed contents
            # page: its chapter headings belong to the contents section,
            # not to chapter starts. (Title matching is unreliable here
            # because the OCR of the contents page adds noise like
            # "CHAPTER IX PAGE".)
            is_contents_page = sum(
                1 for t in data['headings']
                if hocr.structure.is_chapter(t)) > 1
            for item in data['items']:
                if item['kind'] == 'heading' and item['level'] == 1:
                    title = self._clean_heading_title(
                        html.unescape(item['text']))
                    kind = self._section_kind(title)
                    if kind == 'chapter' and is_contents_page:
                        # A contents page's chapter lines belong to the
                        # contents section: they stay in the section's
                        # content as headings but do not start chapter
                        # sections. A front-matter heading on the same
                        # page (CONTENTS) still starts its section.
                        if current is None:
                            current = new_section('preamble', None)
                        current['items'].append(item)
                        current['headings'].append(title)
                        continue
                    if kind:
                        if current:
                            sections.append(current)
                        current = new_section(kind, title)
                        current['items'].append(item)
                        current['headings'].append(title)
                        continue
                if current is None:
                    current = new_section('preamble', None)
                current['items'].append(item)
                if item['kind'] == 'heading' and item['level'] == 1:
                    current['headings'].append(
                        self._clean_heading_title(
                            html.unescape(item['text'])))
        if current:
            sections.append(current)
        # Assign file names: a running counter per kind so a preamble, a
        # front-matter section and chapters never collide.
        counters = {'preamble': 0, 'front': 0, 'chapter': 0}
        for section in sections:
            kind = section['kind']
            counters[kind] += 1
            if kind == 'preamble':
                section['file'] = 'preamble.html'
            elif kind == 'front':
                section['file'] = 'front_%d.html' % counters[kind]
            else:
                section['file'] = 'chapter_%d.html' % counters[kind]
        return sections

    def generate(self, warn_confidence_threshold=75.0, remove_confidence_threshold=None):
        self.book = epub.EpubBook()
        self.book.reset()
        self.set_metadata()
        self._add_page_break_source()

        css_file = epub.EpubItem(
            uid="style_nav",
            file_name="style/style.css",
            media_type="text/css",
            content=self.style
        )
        self.book.add_item(css_file)

        front_matter_epub = epub.EpubHtml(title='Notice', file_name='notice.html', lang='en', media_type='application/xhtml+xml')
        generated_date = datetime.now(timezone.utc).strftime('%Y-%m-%d')
        front_matter_text = self.front_matter.replace(
            '</p></div>',
            '</p><p>This EPUB was generated on %s.</p></div>'
            % generated_date,
            1)
        front_matter_epub.set_content(front_matter_text)

        ocr_system = None

        hocr_ocr_system = hocr.util.get_ocr_system(hocr.util.open_if_required(self.hocr_xml_file_path)) or ''
        hocr_ocr_system = hocr_ocr_system.lower()

        if 'tesseract' in hocr_ocr_system:
            ocr_system = OCR_SYSTEM_TESSERACT
            warning_confidence_threshold = 50
            #remove_confidence_threshold = 30
            remove_confidence_threshold = None
        elif 'abbyy' in hocr_ocr_system or 'luratech' in hocr_ocr_system:
            ocr_system = OCR_SYSTEM_ABBYY
            warning_confidence_threshold = 30
            #remove_confidence_threshold = 20
            remove_confidence_threshold = None
        else:
            ocr_system = OCR_SYSTEM_UNKNOWN
            warning_confidence_threshold = 50
            remove_confidence_threshold = None

        print('Detected OCR system:', ocr_system)

        body_size = self._compute_body_size()

        # Cross-page pass to flag running heads (repeated top-of-page
        # lines) as furniture so they stay out of the prose, the headings
        # and the ToC.
        self.furniture_drop = self._collect_running_heads(body_size)
        if self.furniture_drop:
            print('Running-head furniture: dropping %d lines on %d pages'
                  % (len(self.furniture_drop),
                     len({page for page, _norm in self.furniture_drop})))

        pages_hocr = hocr.parse.hocr_page_iterator(self.hocr_xml_file_path)

        # Iterate all the pages, collecting the ordered content items for
        # each one. One XHTML file is emitted per section (front matter
        # and chapters), so paragraphs can continue across page turns
        # without being split by the file boundary.
        images_found = 0
        words_found = 0
        # Pages whose mean word confidence is below the OCR-system warning
        # threshold (reported in the notice, not in the reading flow), and
        # pages skipped entirely for unreadability.
        self.low_confidence_pages = []
        self.skipped_low_confidence_pages = []
        # Pagebreak markers, for the NCX pageList (older reading systems
        # read only the NCX).
        self.pagebreaks = []
        pages_with_words = 0
        page_data = []
        for page_idx, page in enumerate(pages_hocr):
            if page_idx in self.skip_pages:
                continue
            # Get all the words on the page
            word_data = hocr.parse.hocr_page_to_word_data(page)

            # Get all the photos on the page
            photo_boxes = hocr.parse.hocr_page_to_photo_data(page)

            try:
                _pw, page_height = hocr.parse.hocr_page_get_dimensions(page)
            except Exception:
                page_height = 0

            # Collect blocks as (top_y, html) so horizontal separators can be
            # interleaved with the text in reading order.
            page_items = []
            # Level-1 heading titles on this page, for the table of contents.
            page_headings = []
            page_confidence = 0
            words_on_page = 0
            # ABBYY converter sometimes identifies linebreaks as a negation sign
            hyphens = ['-', '¬']
            # Build the page content while preserving the paragraph structure
            # from the hOCR. Each paragraph becomes its own <p> element, or a
            # heading when it looks like one.
            for element in word_data:
                line_content = []
                for line in element['lines']:
                    # Running heads are furniture: they stay out of the
                    # text, the word/confidence counts and the headings.
                    if self._is_furniture_line(page_idx, line, page_height):
                        continue
                    for word in line['words']:
                        # Save text data
                        text = word['text']
                        text = html.escape(text)
                        if self.strip_whitespaces:
                            text = text.strip()
                        line_content.append(text)

                        # Count word confidence scores
                        page_confidence += word['confidence']
                        words_found += 1
                        words_on_page += 1
                    # Examine the last character of of the last element of the line
                    if len(line_content) and len(line_content[-1]) and line_content[-1][-1] in hyphens:
                        # Remove the last character if it is a hyphen
                        line_content[-1] = line_content[-1][:-1]
                        # Add placeholder value
                        line_content.append('\x7f')

                # Skip paragraphs with no words so we don't emit empty <p> tags.
                if not line_content:
                    continue

                # Join the words of this paragraph, then remove the placeholder
                # (and surrounding spaces) that mark words hyphenated across a
                # line break within this paragraph.
                page_text = ' '.join(line_content)
                page_text = page_text.replace(' \x7f ', '')

                top_y = element['lines'][0]['bbox'][1] if element['lines'] else 0
                fontsize = max((w['fontsize'] for ln in element['lines']
                               for w in ln['words']), default=0.0)
                kind = hocr.structure.classify(page_text, fontsize, body_size)
                if kind[0] == 'heading':
                    level = kind[1]
                    page_items.append(
                        {'kind': 'heading', 'level': level, 'page': page_idx,
                         'top_y': top_y, 'text': page_text})
                    if level == 1:
                        page_headings.append(
                            self._clean_heading_title(html.unescape(page_text)))
                else:
                    page_items.append({'kind': 'para', 'page': page_idx,
                                       'top_y': top_y, 'text': page_text})

            # Horizontal separators become thematic breaks. Skip the running-head
            # underline near the top of the page and any thick non-rule blocks.
            if page_height:
                seps = []
                for sep in page.findall('.//*[@class="ocr_separator"]'):
                    box = hocr.parse.BBOX_REGEX.search(sep.get('title', ''))
                    if not box:
                        continue
                    _x0, y0, _x1, y1 = (int(v) for v in box.group(1).split())
                    if (y1 - y0) <= 20 and y0 > 0.15 * page_height:
                        seps.append(y0)
                # Insert each separator before the first block below it so the
                # paragraph reading order is preserved.
                for y0 in sorted(seps):
                    idx = len(page_items)
                    for i, item in enumerate(page_items):
                        if item['top_y'] > y0:
                            idx = i
                            break
                    page_items.insert(idx, {'kind': 'hr', 'page': page_idx,
                                            'top_y': y0})

            # The pagebreak marker opens the page's content so the page-list
            # can point at it; when the joining pass merges this page's first
            # paragraph with the previous page's last one, the marker moves
            # to the join point inside the merged paragraph.
            marker = self._pagebreak_marker(page_idx)
            if marker:
                page_items.insert(0, {'kind': 'marker', 'page': page_idx,
                                      'top_y': 0, 'html': marker})

            # Add a warning if the confidence in the text is below the given threshold
            if words_on_page:
                page_confidence = page_confidence/words_on_page
                pages_with_words += 1

                if remove_confidence_threshold and page_confidence < remove_confidence_threshold:
                    print('Skipping page due to low confidence: %0.02f%%' % page_confidence)
                    self.skipped_low_confidence_pages.append(
                        (page_idx, page_confidence))
                    continue
                if page_confidence < warning_confidence_threshold:
                    print('Warning for page due to low confidence: %.02f%%' % page_confidence)
                    self.low_confidence_pages.append((page_idx, page_confidence))

            # Add all the images from the page
            images_on_page = 0
            if 'photos' not in self.image_kinds:
                photo_boxes = []
            for image_idx, box in enumerate(photo_boxes):
                # Treat first cover image as special case
                if page_idx not in self.cover_pages[:1]:
                    cropped_image_filename = self.img_stack.crop_image(page_idx, box)
                    if not cropped_image_filename:
                        continue
                    cropped_jpeg_data = open(cropped_image_filename, "rb").read()
                    image_filename_epub = "image_%04u_%02u.jpeg" % (page_idx, image_idx)
                    image_epub = epub.EpubImage()
                    image_epub.file_name = image_filename_epub
                    image_epub.media_type = "image/jpeg"
                    image_epub.set_content(cropped_jpeg_data)

                    self.book.add_item(image_epub)
                    page_items.append({'kind': 'image', 'page': page_idx,
                                       'top_y': 0,
                                       'src': image_filename_epub})
                    images_found += 1
                    images_on_page += 1

            page_data.append({'page': page_idx, 'items': page_items,
                              'headings': page_headings,
                              'has_words': words_on_page > 0,
                              'has_images': images_on_page > 0})

        # Group the pages into sections, one XHTML file each, so
        # paragraphs can continue across page turns.
        sections = self._group_sections(page_data)

        # Emit one file per section with the joining pass, which merges
        # paragraphs that form one sentence across a page turn or an OCR
        # paragraph break.
        pages_epub = [front_matter_epub]
        for section in sections:
            joined = self._join_paragraphs(section['items'])
            section_html = self._render_items(joined)
            if not section_html:
                continue
            section_epub = epub.EpubHtml(
                title=section['title'] or 'Section',
                file_name=section['file'],
                media_type='application/xhtml+xml',
            )
            if self.language:
                section_epub.set_language(self.language)
            else:
                section_epub.omit_lang = True
            section_epub.add_link(
                href='style/style.css', rel='stylesheet', type='text/css'
            )
            section_epub.set_content(section_html)
            pages_epub.append(section_epub)
            for item in section['items']:
                if item['kind'] == 'marker':
                    self.pagebreaks.append(
                        (section['file'], item['page'],
                         self.page_numbers.get(item['page'], '')))

        # Low-confidence pages are reported in the notice instead of
        # interrupting the reading flow on every affected page.
        confidence_notice = self._confidence_notice(pages_with_words)
        if confidence_notice:
            front_matter_epub.set_content(
                front_matter_text.replace(
                    '</p></div>', confidence_notice + '</div>', 1))

        # Add all the sections to the book
        for page_epub in pages_epub:
            self.book.add_item(page_epub)

        # Build the table of contents from the level-1 headings. A section
        # that lists several chapters is the printed contents section, not
        # a chapter start, so skip it; real chapters each open a page with
        # a single heading. (Title matching is unreliable here because the
        # OCR of the contents section adds noise like "CHAPTER IX PAGE".)
        toc = []
        contents_href = None
        for section in sections:
            titles = section['headings']
            toc_heading = next((t for t in titles
                                if hocr.structure.front_matter_type(t) == 'toc'),
                               None)
            if toc_heading and contents_href is None:
                contents_href = section['file']
            if sum(1 for t in titles if hocr.structure.is_chapter(t)) > 1:
                # A section that lists several chapters is the printed
                # contents section, not a chapter start: skip it from the
                # ToC (the 'CONTENTS' heading already carries the 'toc'
                # landmark above).
                continue
            toc.extend((section['file'], t) for t in titles
                       if hocr.structure.is_toc_entry(t))
        self.toc_entries = toc

        self.has_text = words_found > 0
        self.has_images = images_found > 0
        self.set_accessibility_metadata()

        # Set cover. The cover is the book's identity, not a content photo, so
        # it is emitted whenever there is an image stack and a detected cover
        # page, regardless of --images (which only gates the photo loop).
        cover_set = False
        if self.img_stack is not None and len(self.cover_pages):
            image_filename = self.img_stack.crop_image(self.cover_pages[0], None)
            if image_filename:
                jpeg_data = open(image_filename, "rb").read()
                self.book.set_cover("cover.jpeg", jpeg_data)
                cover_set = True

        # Table of contents from the level-1 headings (chapters and front-matter
        # titles), each linking to the page it starts on.
        self.book.toc = [
            epub.Link(href, title, 'toc_%d' % i)
            for i, (href, title) in enumerate(self.toc_entries)
        ]

        # Landmarks: the key entry points a reader wants to jump to. The
        # cover entry points at the cover *page* (cover.xhtml), not the
        # image: the landmark must link to a content document, and
        # epubcheck flags an image reference here (OPF-032) and the
        # unlinked non-linear page (OPF-096).
        guide = []
        if cover_set:
            guide.append({'type': 'cover', 'title': 'Cover',
                          'href': 'cover.xhtml'})
        if contents_href:
            guide.append({'type': 'toc', 'title': 'Contents', 'href': contents_href})
        bodymatter_set = False
        for href, title in self.toc_entries:
            fm = hocr.structure.front_matter_type(title)
            if fm and fm != 'toc':
                guide.append({'type': fm, 'title': title, 'href': href})
            elif not bodymatter_set and hocr.structure.is_chapter(title):
                guide.append({'type': 'bodymatter', 'title': title, 'href': href})
                bodymatter_set = True
        self.book.guide = guide

        self.book.add_item(epub.EpubNcx())
        self.book.add_item(epub.EpubNav())
        # Hand the pagebreak markers to the writer for the NCX pageList.
        self.book.pagebreaks = self.pagebreaks
        # The nav document must not appear in the reading order at all:
        # some readers (e.g. Kobo) ignore hidden="hidden" on <nav> and
        # show it even when it is not in the spine, so the spine carries
        # no nav itemref. epubcheck also requires a non-linear document
        # to be reachable, and the nav is not reachable by design.
        if cover_set:
            self.book.spine = ['cover'] + pages_epub
        else:
            self.book.spine = pages_epub
        write_epub_file(self.epub_zip_file_path, self.book,
                        language=self.language)


if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='hOCR to ePUB converter')

    parser.add_argument('-f', '--infile', help='Item _hocr.html file',
                        type=str, default=None)
    parser.add_argument('-o', '--outfile', help='Output _ebook.epub file',
                        type=str, default=None)
    parser.add_argument('-m', '--metafile', help='Item _meta.xml file',
                        type=str, default=None)
    parser.add_argument('-i', '--imagestack', help='Item imagestack file (usually _jp2.zip)',
                        type=str, default=None)
    parser.add_argument('-s', '--scandata', help='Item _scandata.xml file',
                        type=str, default=None)
    parser.add_argument('-w', '--workingdir', help='Directory used for temp files',
                        type=str, default=None)
    parser.add_argument('--kakadu', help='Use kakadu is available', action='store_true')
    parser.add_argument('--ignore-broken-images', help='Continue even if kakadu cannot use an image',
                        action='store_true')
    parser.add_argument('--images', help='Image kinds to embed (repeatable): "photos" for '
                        'per-page illustrations, "cover" for the cover image. Default: none.',
                        action='append', choices=['photos', 'cover'], default=None)
    args = parser.parse_args()

    if not args.infile:
        raise Exception("Must provide hOCR input file with -f")

    # Allow external caller to override working directory from default /tmp/ or /var/tmp/fast/
    if args.workingdir:
        WORKING_DIR = args.workingdir

    EpubGenerator(
        args.infile,
        args.metafile,
        args.imagestack,
        args.scandata,
        args.outfile,
        use_kakadu=args.kakadu,
        ignore_broken_images=args.ignore_broken_images,
        image_kinds=set(args.images or []))
