306 lines
12 KiB
Ruby
306 lines
12 KiB
Ruby
# Compares every translation of the three books with the English, block by block, so a translation
|
|
# that has fallen behind the English, or lost or broken something on the way, is found before it
|
|
# ships. The books are docs/Manual.adoc, docs/Reference.adoc, and docs/Learn.adoc (with the lessons
|
|
# it includes); each language has the same files under docs/<language>/. For each it checks:
|
|
#
|
|
# * the same sections, with the same ids, levels, and kinds, in the same order (a translation
|
|
# writes the English id of every section whose id Asciidoctor would make from its title);
|
|
# * within each section, the same blocks in the same order: paragraphs, lists with as many items,
|
|
# tables with as many rows and columns, the same pictures, notes of the same kind, and listings
|
|
# in the same language;
|
|
# * Lua listings identical once their comments are taken out (only comments are translated), and
|
|
# every other listing with as many lines;
|
|
# * within each section, the same references (<<id>> and xref:Book.adoc#id), as many times each;
|
|
# * that Asciidoctor finds nothing wrong in the translation it does not find in the English;
|
|
# * that the English is ASCII and the translation names its language (:lang:) and has its captions.
|
|
#
|
|
# Inline code (`spans`) and web addresses that differ within a section are warnings; everything else
|
|
# is an error. Exits 1 on any error.
|
|
#
|
|
# ruby util/checkTranslations.rb [language...] every language, or those named
|
|
# ruby util/checkTranslations.rb --pair english.adoc translated.adoc
|
|
# two pieces of a book, as a
|
|
# translator checks a part
|
|
require 'asciidoctor'
|
|
|
|
LANGUAGES = %w[fr it de es]
|
|
BOOKS = %w[Manual Reference Learn]
|
|
REPO = File.dirname(__dir__)
|
|
DOCS = File.join(REPO, 'docs')
|
|
REPORTED = 12 # Differences reported per section before the rest are only counted
|
|
|
|
|
|
# Asciidoctor's warnings for a document, without their line numbers and with only the last part of
|
|
# any path, so the English's and a translation's can be compared.
|
|
def warningTexts(logger)
|
|
logger.messages.select { |entry| entry[:severity] != :INFO && entry[:severity] != :DEBUG }.map do |entry|
|
|
text = entry[:message]
|
|
text = text[:text] if text.is_a?(Hash)
|
|
text.to_s.gsub(%r{(?:[^\s:]*/)+([^\s/]+)}, '\1')
|
|
end
|
|
end
|
|
|
|
|
|
# Loads a book or a piece of one with every block's source line, and its warnings.
|
|
def loadBook(path)
|
|
logger = Asciidoctor::MemoryLogger.new
|
|
Asciidoctor::LoggerManager.logger = logger
|
|
doc = Asciidoctor.load_file(path, safe: :unsafe, sourcemap: true, attributes: { 'revnumber' => 'x', 'doctype' => 'book' })
|
|
[doc, warningTexts(logger)]
|
|
end
|
|
|
|
|
|
# A Lua listing with its comments taken out and blank and trailing space dropped: what a
|
|
# translation must leave exactly as it is.
|
|
def luaWithoutComments(code)
|
|
out = +''
|
|
index = 0
|
|
while index < code.length
|
|
char = code[index]
|
|
if char == '-' && code[index + 1] == '-'
|
|
long = code[(index + 2)..].match(/\A\[(=*)\[/)
|
|
if long
|
|
close = "]#{long[1]}]"
|
|
found = code.index(close, index + 2 + long[0].length)
|
|
index = found ? found + close.length : code.length
|
|
else
|
|
found = code.index("\n", index)
|
|
index = found || code.length
|
|
end
|
|
elsif char == '"' || char == "'"
|
|
found = index + 1
|
|
while found < code.length && code[found] != char && code[found] != "\n"
|
|
found += (code[found] == '\\') ? 2 : 1
|
|
end
|
|
out << code[index..found].to_s
|
|
index = found + 1
|
|
elsif char == '[' && (long = code[index..].match(/\A\[(=*)\[/))
|
|
close = "]#{long[1]}]"
|
|
found = code.index(close, index + long[0].length)
|
|
found = found ? found + close.length : code.length
|
|
out << code[index...found]
|
|
index = found
|
|
else
|
|
out << char
|
|
index += 1
|
|
end
|
|
end
|
|
out.lines.map(&:rstrip).reject(&:empty?).join("\n")
|
|
end
|
|
|
|
|
|
# What a translation must keep of a block.
|
|
def blockKey(node)
|
|
key = node.context.to_s
|
|
case node.context
|
|
when :section
|
|
key = "section #{node.level} #{node.id} #{node.sectname}"
|
|
when :listing, :literal
|
|
language = node.attr('language')
|
|
if language == 'lua'
|
|
key = "#{node.context} lua\n#{luaWithoutComments(node.source)}"
|
|
else
|
|
key = "#{node.context} #{language} #{node.source.lines.count} lines"
|
|
end
|
|
when :image
|
|
key = "image #{node.attr('target')}"
|
|
when :table
|
|
key = "table #{node.rows.head.size}+#{node.rows.body.size}+#{node.rows.foot.size} rows #{node.columns.size} columns"
|
|
when :ulist, :olist, :colist, :dlist
|
|
key = "#{node.context} of #{node.items.size}"
|
|
when :admonition
|
|
key = "admonition #{node.attr('name')}"
|
|
when :floating_title
|
|
key = "heading #{node.level}"
|
|
end
|
|
if node.context != :section && node.id && !node.id.start_with?('_')
|
|
key += " ##{node.id}"
|
|
end
|
|
key
|
|
end
|
|
|
|
|
|
# Every block of a document in order, grouped by the section it is in. Returns the sections' ids
|
|
# in order and, for each, its blocks' keys and source places.
|
|
def sections(doc)
|
|
order = ['(top)']
|
|
groups = { '(top)' => [] }
|
|
places = { '(top)' => [] }
|
|
starts = { '(top)' => nil }
|
|
current = '(top)'
|
|
doc.find_by(traverse_documents: true).each do |node|
|
|
next if node.context == :document || node.context == :list_item
|
|
if node.context == :section
|
|
current = node.id
|
|
order << current
|
|
groups[current] = []
|
|
places[current] = []
|
|
starts[current] = node.source_location
|
|
end
|
|
groups[current] << blockKey(node)
|
|
places[current] << node.source_location
|
|
end
|
|
[order, groups, places, starts]
|
|
end
|
|
|
|
|
|
# The text of each section as written, from its title to the next section's in the same file,
|
|
# without its listings: where its references and inline code are counted.
|
|
def sectionTexts(starts)
|
|
texts = {}
|
|
byFile = starts.select { |_, place| place }.group_by { |_, place| place.file }
|
|
byFile.each do |file, list|
|
|
next unless file && File.exist?(file)
|
|
lines = File.readlines(file)
|
|
list = list.sort_by { |_, place| place.lineno }
|
|
list.each_with_index do |(id, place), index|
|
|
last = (index + 1 < list.size) ? list[index + 1][1].lineno - 1 : lines.size
|
|
inBlock = nil
|
|
text = +''
|
|
lines[(place.lineno - 1)...last].each do |line|
|
|
fence = line.rstrip
|
|
if fence.match?(/\A(-{4,}|\.{4,}|\+{4,})\z/)
|
|
inBlock = (inBlock == fence) ? nil : (inBlock || fence)
|
|
next
|
|
end
|
|
text << line unless inBlock
|
|
end
|
|
texts[id] = text
|
|
end
|
|
end
|
|
texts
|
|
end
|
|
|
|
|
|
def counts(list)
|
|
list.each_with_object(Hash.new(0)) { |item, total| total[item] += 1 }
|
|
end
|
|
|
|
|
|
# The references, inline code, and web addresses of a section's text.
|
|
def inlineParts(text)
|
|
references = text.scan(/<<([^,>]+)/).flatten.map(&:strip) + text.scan(/xref:([^\[\s]+)\[/).flatten
|
|
# Asciidoctor's own rule for constrained monospace, so a span it will not render (l'`id`, an
|
|
# apostrophe before the backtick) is not counted as one.
|
|
code = text.scan(/(?:^|(?<=[^\p{Word};:"'`}\\+]))`(\S|\S[^`]{0,200}?\S)`(?![\p{Word}"'`])/m).flatten.map { |span| span.gsub(/\s+/, ' ') }
|
|
addresses = text.scan(%r{https?://[^\s\[\]<>"]+}).map { |address| address.sub(/[.,;:)]+\z/, '') }
|
|
[counts(references), counts(code), counts(addresses)]
|
|
end
|
|
|
|
|
|
def where(place)
|
|
place ? "#{place.file.sub("#{REPO}/", '')}:#{place.lineno}" : '?'
|
|
end
|
|
|
|
|
|
def describe(key)
|
|
first = key.lines.first.to_s.strip
|
|
key.include?("\n") ? "#{first} (#{key.lines.count - 1} lines of code)" : first
|
|
end
|
|
|
|
|
|
def differences(english, translated)
|
|
(english.keys | translated.keys).filter_map do |item|
|
|
gap = translated[item] - english[item]
|
|
gap.zero? ? nil : "#{item} #{gap.positive? ? '+' : ''}#{gap}"
|
|
end
|
|
end
|
|
|
|
|
|
# Compares one book or piece with its translation. Returns the errors and warnings.
|
|
def compare(englishPath, translatedPath, language)
|
|
errors = []
|
|
warnings = []
|
|
english, englishWarnings = loadBook(englishPath)
|
|
translated, translatedWarnings = loadBook(translatedPath)
|
|
|
|
File.read(englishPath).each_line.with_index do |line, index|
|
|
errors << "#{where(Asciidoctor::Reader::Cursor.new(englishPath, nil, nil, index + 1))}: the English is not ASCII" unless line.ascii_only?
|
|
end
|
|
if language
|
|
errors << "#{translatedPath}: :lang: is #{translated.attr('lang').inspect}, not #{language}" if translated.attr('lang') != language
|
|
errors << "#{translatedPath}: the captions are English; include::locale.adoc[] is missing" if translated.attr('toc-title') == english.attr('toc-title')
|
|
end
|
|
gap = counts(translatedWarnings)
|
|
counts(englishWarnings).each { |text, count| gap[text] -= count }
|
|
gap.each { |text, count| errors << "#{translatedPath}: Asciidoctor warns, where the English does not: #{text}" if count.positive? }
|
|
|
|
englishOrder, englishGroups, englishPlaces, englishStarts = sections(english)
|
|
translatedOrder, translatedGroups, translatedPlaces, translatedStarts = sections(translated)
|
|
missing = englishOrder - translatedOrder
|
|
extra = translatedOrder - englishOrder
|
|
missing.each { |id| errors << "#{where(englishStarts[id])}: section #{id} is not in the translation" }
|
|
extra.each { |id| errors << "#{where(translatedStarts[id])}: section #{id} is not in the English" }
|
|
if missing.empty? && extra.empty? && englishOrder != translatedOrder
|
|
at = englishOrder.zip(translatedOrder).index { |a, b| a != b }
|
|
errors << "#{where(translatedStarts[translatedOrder[at]])}: the sections are in a different order from here (the English has #{englishOrder[at]})"
|
|
end
|
|
|
|
englishTexts = sectionTexts(englishStarts)
|
|
translatedTexts = sectionTexts(translatedStarts)
|
|
(englishOrder & translatedOrder).each do |id|
|
|
ours = englishGroups[id]
|
|
theirs = translatedGroups[id]
|
|
if ours != theirs
|
|
at = (0...[ours.size, theirs.size].max).find { |index| ours[index] != theirs[index] }
|
|
if ours[at] && theirs[at] && ours[at].include?("\n") && theirs[at].include?("\n")
|
|
line = ours[at].lines.zip(theirs[at].lines).index { |a, b| a != b } || 0
|
|
detail = "the code differs at its line #{line}: English #{ours[at].lines[line].to_s.strip.inspect}, translation #{theirs[at].lines[line].to_s.strip.inspect}"
|
|
else
|
|
detail = "English has #{ours[at] ? describe(ours[at]) : 'nothing more'} (#{where(englishPlaces[id][at])}), translation has #{theirs[at] ? describe(theirs[at]) : 'nothing more'}"
|
|
end
|
|
errors << "#{where(translatedPlaces[id][at] || translatedStarts[id])}: in section #{id}, block #{at + 1}: #{detail}"
|
|
end
|
|
next unless englishTexts[id] && translatedTexts[id]
|
|
ourReferences, ourCode, ourAddresses = inlineParts(englishTexts[id])
|
|
theirReferences, theirCode, theirAddresses = inlineParts(translatedTexts[id])
|
|
gaps = differences(ourReferences, theirReferences)
|
|
errors << "#{where(translatedStarts[id])}: in section #{id}, references differ: #{gaps.join(', ')}" unless gaps.empty?
|
|
gaps = differences(ourCode, theirCode)
|
|
warnings << "#{where(translatedStarts[id])}: in section #{id}, inline code differs: #{gaps.first(REPORTED).join(', ')}#{gaps.size > REPORTED ? ", and #{gaps.size - REPORTED} more" : ''}" unless gaps.empty?
|
|
gaps = differences(ourAddresses, theirAddresses)
|
|
warnings << "#{where(translatedStarts[id])}: in section #{id}, web addresses differ: #{gaps.join(', ')}" unless gaps.empty?
|
|
end
|
|
[errors, warnings]
|
|
end
|
|
|
|
|
|
def report(errors, warnings)
|
|
errors.each { |text| puts "error: #{text}" }
|
|
warnings.each { |text| puts "warning: #{text}" }
|
|
end
|
|
|
|
|
|
def main
|
|
errors = []
|
|
warnings = []
|
|
if ARGV.first == '--pair'
|
|
abort 'usage: checkTranslations.rb --pair english.adoc translated.adoc' unless ARGV.size == 3
|
|
errors, warnings = compare(File.expand_path(ARGV[1]), File.expand_path(ARGV[2]), nil)
|
|
else
|
|
languages = ARGV.empty? ? LANGUAGES : ARGV
|
|
languages.each do |language|
|
|
abort "unknown language #{language}" unless LANGUAGES.include?(language)
|
|
lessons = Dir.glob(File.join(DOCS, 'lessons', '*.adoc')).map { |path| File.join('lessons', File.basename(path)) }
|
|
lessons.each do |lesson|
|
|
errors << "docs/#{language}/#{lesson}: missing" unless File.exist?(File.join(DOCS, language, lesson))
|
|
end
|
|
BOOKS.each do |book|
|
|
translated = File.join(DOCS, language, "#{book}.adoc")
|
|
if File.exist?(translated)
|
|
found = compare(File.join(DOCS, "#{book}.adoc"), translated, language)
|
|
errors.concat(found[0])
|
|
warnings.concat(found[1])
|
|
else
|
|
errors << "docs/#{language}/#{book}.adoc: missing"
|
|
end
|
|
end
|
|
end
|
|
end
|
|
report(errors, warnings)
|
|
puts "#{errors.size} errors, #{warnings.size} warnings"
|
|
exit(errors.empty? ? 0 : 1)
|
|
end
|
|
|
|
|
|
main
|