gwt_corpus.py | TOKENIZE = 'unicode61 remove_diacritics 2' | TOKENIZE = 'unicode61 remove_diacritics 0' | a search folds accents: «jose» finds «José»
gwt_corpus.py | return 'body : (NEAR(%s, %d))' % (' '.join('"%s"' % w for w in ws), near) | return 'body : (%s)' % ' OR '.join('"%s"' % w for w in ws) | a search needs every word, never OR
gwt_corpus.py | return WORD.findall(text or '') | return (text or '').split() | a + or a quote in a query separates words that must all appear, never a phrase or an FTS5 syntax error
gwt_corpus.py |         if not run_query(db, expr, a.source, 1): |         if False: | --fts with a control that finds nothing is void, exit 3
gwt_corpus.py |     if a.fts and not a.control: |     if False: | --fts without a known-positive control is refused
gwt_corpus.py |         return EXIT_NONE |         return EXIT_HITS | 0 hits exits 1 and says «not downloaded yet», never success
gwt_corpus.py |     if not controls('before'): |     if False: | a sweep whose control is missing before it starts is void and writes no ledger row
gwt_corpus.py |     if not controls('after'): |     if False: | a sweep whose control is lost by the end is void and writes no ledger row
gwt_corpus.py |             extra.append('%s %s' % (right, left.title())) |             pass | an inverted dictionary heading «SURNAME, Given» matches the full name
gwt_corpus.py |             if not short or len(where) <= max_pair_pages: |             if True: | a surname pair or a two-word name on more than --max-pair-pages pages gives no match
gwt_corpus.py |             if carriers[s] > max_surname_people: |             if False: | a surname that many people carry gives no surname-and-place match
gwt_corpus.py | if path not in seen and any(path.startswith(r + os.sep) for r in roots): | if path not in seen: | an update of one root never drops the files of another
gwt_corpus.py |                 if old and old[0] == st.st_mtime and old[1] == st.st_size: |                 if old: | a file whose size or time changed is read again
gwt_corpus.py |             return None  # its metadata is not written yet |             return []  # its metadata is not written yet | a cache body whose metadata is not written yet is retried at the next update
gwt_corpus.py | s = re.sub(r'(?is)<(script|style)\b.*?</\1>', ' ', s) | s = s | a page's scripts and styles are not indexed as its text
gwt_corpus.py |             return []  # an fscache entry's metadata | pass  # an fscache entry's metadata | a reader's candidates file is not indexed twice
gwt_corpus.py |             ctx['no_pdftotext'] += 1 |             pass | an update that could not read PDFs says how many
gwt_corpus.py | return WORD.findall(text or '') | return rules.fold(text).split() | a query word with ß matches the same word in the index; Python's fold would turn it into ss
gwt_corpus.py |             return None  # not recorded: the next update tries it again |             return []  # not recorded: the next update tries it again | a PDF pdftotext failed on is read again at the next update
gwt_corpus.py |             return None  # not recorded: read once pdftotext is installed |             return []  # not recorded: read once pdftotext is installed | a PDF met before pdftotext was installed is read once it is
gwt_corpus.py |     if r.returncode: |     if False: | a pdftotext that fails is a failed read, never a PDF with no words
gwt_corpus.py | 'final_url', 'query', 'q', 'params', 'responseHeader'} | 'final_url'} | a 0-result answer that repeats the query never matches the name it was asked for
gwt_corpus.py |         return 'body : ("%s")' % ' '.join(ws) |         return '"%s"' % ' '.join(ws) | a search looks at the page text, never at the request URL that holds the name asked for
gwt_corpus.py |             return []  # the tree itself |             pass  # the tree itself | a tree dump under a root is not indexed, so people never find themselves
gwt_corpus.py | isinstance(o.get('entries'), list) and len(o['entries']) > 1 | isinstance(o.get('entries'), list) and len(o['entries']) > 999 | each entry of a search answer is its own row with its own record link
gwt_corpus.py |             except (OSError, ValueError, AttributeError, TypeError) as e: |             except ZeroDivisionError as e: | one unreadable file is counted and skipped, never the end of the run
gwt_corpus.py |     return pages[:-1] if len(pages) > 1 and not pages[-1].strip() else pages |     return pages | the empty page after a final form feed is not a page
gwt_corpus.py | decode('utf-8-sig', 'replace') | decode('utf-8', 'replace') | a names TSV saved with a byte order mark is read
gwt_corpus.py | .replace('\r\n', '\n') | .replace('\r\n', '\r\n') | a word broken over a Windows line end is joined
gwt_corpus.py |     if any(cols.index(c) >= len(cells) for c in ('key', 'given', 'surname')): |     if False: | a names row short of columns is bad input, never a traceback
gwt_corpus.py |             gz = f.read(2) == b'\x1f\x8b' |             gz = False | a gzipped dump is read
gwt_corpus.py |     real = [i for i, w in enumerate(sur) if rules.fold(w) not in PARTICLES] |     real = list(range(len(sur))) | a particle (de, la, van) is never a surname of its own for pairs and places
gwt_corpus.py |             if short and len(found) > a.max_pages: |             if False: | a two-word phrase on more than --max-pages pages is no lead in a sweep
gwt_corpus.py |                 if len(where) <= max_place_pages: |                 if True: | a surname and town on more than --max-place-pages pages gives no match
gwt_corpus.py | row.update(result='partial', | row.update(result='hit', | a corpus match is a lead to read (partial), never a finding
gwt_corpus.py | row.update(result='none', evidence='corpus:%s' % os.path.basename(a.db), next_date=today) | row.update(result='none', evidence='corpus:%s' % os.path.basename(a.db)) | a corpus miss is due again at once and never marks the person as searched
gwt_corpus.py | 'agent': a.agent, 'control': run_id} | 'agent': a.agent} | sweep rows name the run whose controls passed
gwt_corpus.py |         have = {(r['person_key'], r['source'], r['query'], r['result'], r['date']) for r in led.read_rows(a.ledger)} |         have = set() | the same sweep twice on one day adds no rows
gwt_corpus.py |     if n < 1: |     if False: | a --limit below 1 is refused
gwt_corpus.py |             raise BadInput('%s: pdftotext could not read it (%s)' % (path, e)) |             pages = [] | xref on a PDF pdftotext cannot read says so
gwt_corpus.py |         ctx['pdf_no_text'] += 1 |         pass | an update says how many PDFs have no text layer (scans to OCR)
gwt_corpus.py | except (OSError, subprocess.TimeoutExpired) as e: | except ZeroDivisionError as e: | a pdftotext that times out is a failed read, tried again at the next update
gwt_corpus.py |             if (n['new'] + n['changed']) % 1000 == 0: |             if False: | a long first index keeps what it has read if it stops before the end
gwt_corpus.py |             raise BadInput('%s has no text layer; OCR it first (see ocr-bench)' % path) |             pass | xref on a PDF with no text layer says so, never «0 matches»
gwt_corpus.py | if ' %s ' % needle in pages[i - 1]) | if True) | xref needs the words of a name side by side, not anywhere on the page
gwt_corpus.py |         raise BadInput('--dump %s: not a gwt-check dump (no objects.people)' % path) |         pass | a --dump that is not a dump is bad input, never «0 matches»
gwt_corpus.py |             r['label'] = r['label'] or ('%s %s' % (r['given'], r['surname'])).strip() |             pass | a names row with an empty label is labelled with the name
gwt_corpus.py | evidence='%s%s' % (first[2] or first[3], | evidence='%s%s' % ('', | a sweep lead names the URL or file to read
gwt_corpus.py | fcntl.flock(lock, | (lock, | a second update exits 2 while the first holds the lock
gwt_corpus.py | return EXIT_HITS if hits else EXIT_NONE | return EXIT_HITS | a sweep that finds nobody exits 1, never success
gwt_corpus.py | None, max(a.limit, a.max_pages + 1)) | None, a.limit) | a sweep with a small --limit still counts the pages of a two-word phrase before it calls it rare
gwt_corpus.py | if i not in done: | if True: | xref gives at most one match per person and page
gwt_corpus.py | '.pdf', '.tsv', '.csv', '.eml'} | '.pdf'} | a .tsv, .csv or .eml under a root is indexed, never passed over in silence
gwt_corpus.py |                  '') for i, line in enumerate(lines)] |                  '') for i, line in enumerate([text])] | a TSV or CSV is one row per line, so a hit points at its row
gwt_corpus.py |         skip = {j for j, c in enumerate(cols) if c in REQUEST_COLUMNS} |         skip = set() | a search table's query column is not indexed: a row never matches the name it was asked for
gwt_corpus.py |         return next(csv.reader([line]), []) |         return line.split(',') | a quoted CSV cell holding a comma keeps the columns in line, so the query column is still left out
gwt_corpus.py |     except csv.Error:  # a cell |     except ZeroDivisionError:  # a cell | a CSV cell over the csv size limit is still read, and the update goes on
gwt_corpus.py |         if 'person_key' in cols: |         if False: | a gwt-ledger TSV is not indexed: a person never finds the query asked for them
gwt_corpus.py |     out = ['%s: %s' % (h, v) for h in EML_HEADERS for v in [eml_header(msg, h)] if v] |     out = [] | a mail's encoded From and Subject are decoded and indexed
gwt_corpus.py |     except (IndexError, AttributeError): |     except ZeroDivisionError: | a mail whose From has an unclosed quote is still read, its raw From too, and the update goes on
gwt_corpus.py |         return ' '.join(v for k, v in msg.raw_items() if k.lower() == h.lower()) |         return '' | a header Python cannot parse is indexed as its raw text
gwt_corpus.py |     except IndexError:  # a broken Content-Type |     except ZeroDivisionError:  # a broken Content-Type | a mail whose Content-Type Python cannot parse is still read, and the update goes on
gwt_corpus.py | if p.get_content_maintype() == 'text'] | if True] | a mail attachment that is not text is never indexed
gwt_corpus.py |             if not plain: |             if True: | the HTML copy of a mail's plain part is not read twice
gwt_corpus.py |                 out.append(strip_html(part_text(p))) |                 out.append(part_text(p)) | a mail's HTML part is read as text, without tags or scripts
gwt_corpus.py | raw.decode(part.get_content_charset() or 'utf-8', 'replace') | raw.decode('utf-8', 'replace') | a mail part is decoded in its own charset
gwt_corpus.py |     except LookupError: |     except ZeroDivisionError: | a mail part with an unknown charset is read as UTF-8, never an unreadable file
gwt_corpus.py | if k not in SKIP_KEYS or (k == 'links' and isinstance(v, list)): | if k not in SKIP_KEYS: | a list of hits under links is read, in a reader envelope too
gwt_corpus.py | if k not in SKIP_KEYS or (k == 'links' and isinstance(v, list)): | if k not in SKIP_KEYS or k == 'links': | a GEDCOM X links object is still not indexed
gwt_corpus.py |         if isinstance(o, dict) and isinstance(o.get('links'), list): |         if False: | each hit of a links list is its own row with its own record link
gwt_corpus.py |     for k in ('h', 'href', 'url'): |     for k in ('href', 'url'): | a links hit's short h key is its record link
gwt_corpus.py |                      0, '') for h in o['links']] + [('file', name, ' · '.join(json_strings(rest, [])), url, 0, '')] |                      0, '') for h in o['links']] | the rest of a links file is still indexed
gwt_corpus.py |         skip = {j for j, c in enumerate(cols) if c in REQUEST_COLUMNS} |         skip = {j for j, c in enumerate(cols) if c in SKIP_KEYS} | a table's principal column, a record's name, is still read
