Switch to generic .po[t] handling

Signed-off-by: Stephen Gallagher <sgallagh@redhat.com>

Author: sgallagher

mmdzanata/__init__.py

@@ -196,34 +196,20 @@ def get_module_catalog_from_tags(session, tags, debug=False):
     return catalog
 
 
-def get_translated_locales(zanata_rest_url, zanata_project,
-                           zanata_project_version,
-                           debug=False):
-    # Get the statistics on the translation project for this version
-    stats_url = zanata_rest_url + "/stats/proj/%s/iter/%s" % (
-        zanata_project, zanata_project_version)
-
-    r = requests.get(stats_url, headers={"Accept": "application/json"})
-    if r.status_code == 404:
-        print("Project '%s:%s' does not exist." % (
-            zanata_project, zanata_project_version),
-            file=sys.stderr)
-        raise NonexistentProjectError(zanata_project, zanata_project_version)
-    elif r.status_code != 200:
-        raise UnexpectedHTTPResponse(r.status_code, r.content)
-
-    # We will pull down information for any locale that is at least partially
-    # translated
-    translated_locales = [t["locale"]
-                          for t in r.json()['stats']
-                          if t["translated"] > 0]
+def split_location(location):
+    location_data = location.split(';')
+    if len(location_data) < 3 or len(location_data) > 5:
+        print("Invalid location clue in translation data: %s" % (
+            location), file=sys.stderr)
 
-    if debug:
-        print("Available locales: ")
-        for locale in translated_locales:
-            print("* %s" % locale)
+    module_name = location_data[0]
+    module_stream = location_data[1]
+    profile_name = None
+    translation_type = location_data[2]
+    if translation_type == 'profile':
+        profile_name = location_data[3]
 
-    return translated_locales
+    return module_name, module_stream, profile_name, translation_type
 
 
 def get_modulemd_translations_from_catalog_dict(catalog_dict):
@@ -248,30 +234,25 @@ def get_modulemd_translations_from_catalog_dict(catalog_dict):
                 continue
 
             for location, _ in msg.locations:
-                split_location = location.split(';')
-                if len(split_location) < 3 or len(split_location) > 5:
-                    print("Invalid location clue in translation data: %s" % (
-                        location), file=sys.stderr)
-
-                module_name = split_location[0]
-                module_stream = split_location[1]
+                (module_name, module_stream, profile_name, translation_type)\
+                    = split_location(location)
 
                 try:
                     entry = entries[(module_name, module_stream, locale)]
                 except KeyError:
                     entry = Modulemd.TranslationEntry.new(locale)
 
                 # Summary Translation
-                if split_location[2] == "summary":
+                if translation_type == "summary":
                     entry.set_summary(msg.string)
 
                 # Description Translation
-                elif split_location[2] == "description":
+                elif translation_type == "description":
                     entry.set_description(msg.string)
 
                 # Translation of profile descriptions
-                elif split_location[2] == "profile":
-                    entry.set_profile_description(split_location[3],
+                elif translation_type == "profile":
+                    entry.set_profile_description(profile_name,
                                                   msg.string)
 
                 entries[(module_name, module_stream, locale)] = entry
@@ -300,38 +281,17 @@ def get_modulemd_translations_from_catalog_dict(catalog_dict):
     return mmd_translations.values()
 
 
-def get_modulemd_translations(zanata_rest_url, zanata_project,
-                              os_branch, zanata_translation_file,
+def get_modulemd_translations(translation_files,
                               debug=False):
-    translated_locales = get_translated_locales(zanata_rest_url,
-                                                zanata_project,
-                                                os_branch,
-                                                debug)
 
     catalogs = dict()
-    for loc in translated_locales:
-        # Get the translation data for this locale
-        pofile_url = zanata_rest_url + \
-            "/file/translation/%s/%s/%s/po?docId=%s" % (
-                zanata_project, os_branch, loc,
-                zanata_translation_file)
-        r = requests.get(pofile_url,
-                         headers={"Accept": "application/octet-stream"})
-        if r.status_code != 200:
-            print("Could not retrieve translations for %s" % loc,
-                  file=sys.stderr)
-            continue
-
-        if debug:
-            print("PO content for locale '%s'" % loc)
-            print(r.text)
-
-        # Read the po file into a catalog, indexed by the locale
-        catalogs[loc] = pofile.read_po(
-            BytesIO(r.content),
-            domain=zanata_translation_file)
+    for f in translation_files:
+        with open(f, 'r') as infile:
+            catalog = pofile.read_po(infile)
+            catalogs[catalog.locale_identifier] = catalog
 
     translations = get_modulemd_translations_from_catalog_dict(catalogs)
+
     if debug:
         for translation in translations:
             print(translation.dumps())

mmdzanata/cli.py

@@ -16,9 +16,8 @@
 import click
 import gi
 import koji
-import mmdzanata
-import mmdzanata.fedora
 import os
+import os.path
 import subprocess
 import sys
 import shutil
@@ -29,6 +28,9 @@
 gi.require_version('Modulemd', '1.0')
 from gi.repository import Modulemd
 
+from mmdzanata import get_module_catalog_from_tags, get_modulemd_translations
+from mmdzanata.fedora import get_fedora_rawhide_version, \
+    get_tags_for_fedora_branch, KOJI_URL
 
 ##############################################################################
 # Common options for all commands                                            #
@@ -38,36 +40,15 @@
 @click.group()
 @click.option('--debug/--no-debug', default=False)
 @click.option('-k', '--koji-url',
-              default=mmdzanata.fedora.KOJI_URL,
+              default=KOJI_URL,
               type=str, help="The URL of the Koji build system.",
               show_default=True,
               metavar="<URL>")
 @click.option('-b', '--branch', default="rawhide", type=str,
               help="The distribution release",
               metavar="<branch_name>")
-@click.option('-z', '--zanata-url',
-              default=mmdzanata.fedora.ZANATA_URL,
-              type=str, help="The Zanata URL",
-              show_default=True,
-              metavar="<zanata_project>")
-@click.option('-p', '--zanata-project',
-              default=mmdzanata.fedora.ZANATA_PROJECT,
-              type=str, help="The Zanata project",
-              show_default=True,
-              metavar="<zanata_project>")
-@click.option('-f', '--zanata-translation-document',
-              default=mmdzanata.fedora.ZANATA_DOCUMENT,
-              help="The name of the translated file in Zanata.",
-              show_default=True,
-              metavar="<translation_document>")
-@click.option('-c', '--zanata-user-config',
-              default=lambda: "%s/.config/zanata.ini" % (
-                      os.environ.get("HOME", '~')),
-              help="Path to the Zanata User Config INI file",
-              type=click.Path(exists=True))
 @click.pass_context
-def cli(ctx, debug, branch, koji_url, zanata_url, zanata_project,
-        zanata_translation_document, zanata_user_config):
+def cli(ctx, debug, branch, koji_url):
     """Tools for managing modularity translations."""
 
     ctx.obj = dict()
@@ -78,13 +59,7 @@ def cli(ctx, debug, branch, koji_url, zanata_url, zanata_project,
     ctx.obj['branch'] = branch
 
     if branch == "rawhide":
-        ctx.obj['branch'] = mmdzanata.fedora.get_fedora_rawhide_version(
-            ctx.obj['session'])
-
-    ctx.obj['zanata_url'] = zanata_url
-    ctx.obj['zanata_project'] = zanata_project
-    ctx.obj['zanata_translation_document'] = zanata_translation_document
-    ctx.obj['zanata_user_config'] = zanata_user_config
+        ctx.obj['branch'] = get_fedora_rawhide_version(ctx.obj['session'])
 
 ##############################################################################
 # Subcommands                                                                #
@@ -96,114 +71,97 @@ def cli(ctx, debug, branch, koji_url, zanata_url, zanata_project,
 
 
 @cli.command()
-@click.option('--upload/--no-upload', default=False,
-              help='Whether to automatically push extracted strings to '
-              'Zanata',
-              show_default=True)
+@click.option('-p', '--pot-file',
+              default='fedora-modularity-translations.pot',
+              type=click.File(mode='wb', atomic=True, lazy=True),
+              show_default=True,
+              metavar="<PATH>",
+              help="Path to the portable object template (POT) file to hold "
+                   "the translatable strings.")
 @click.pass_context
-def extract(ctx, upload):
+def extract(ctx, pot_file):
     """
     Extract translatable strings from modules.
 
     Extract translations from all modules included in a particular version of
     Fedora or EPEL.
     """
 
-    catalog = mmdzanata.get_module_catalog_from_tags(
-        ctx.parent.obj['session'], mmdzanata.fedora.get_tags_for_fedora_branch(
+    catalog = get_module_catalog_from_tags(
+        ctx.parent.obj['session'], get_tags_for_fedora_branch(
             ctx.parent.obj['branch']),
         debug=ctx.parent.obj['debug'])
 
-    # Create a temporary directory to hold data while we generate it
-    tdir = tempfile.mkdtemp()
-
-    po_basename = "%s.pot" % ctx.parent.obj['zanata_translation_document']
-    potfile = "%s/%s" % (tdir, po_basename)
-
-    with open(potfile, mode="wb") as f:
-        pofile.write_po(f, catalog, sort_by_file=True)
-
-    # Optionally upload the extracted strings directly to Zanata
-    if upload:
-        # Use the zanata-cli to upload the pot file
-        # It would be better to use the REST API directly here, but the XML
-        # payload format is not documented.
-
-        # First ensure that the requested branch exists in Zanata
-        zanata_args = [
-            '/usr/bin/zanata-cli', '-B', '-e', 'put-version',
-            '--url', ctx.parent.obj['zanata_url'],
-            '--version-project', ctx.parent.obj['zanata_project'],
-            '--version-slug', ctx.parent.obj['branch'],
-            '--user-config', ctx.parent.obj['zanata_user_config']
-        ]
-        result = subprocess.run(zanata_args, capture_output=True)
-        if result.returncode or ctx.parent.obj['debug']:
-            print(result.stderr.decode('utf-8'))
-            print(result.stdout.decode('utf-8'))
-        if result.returncode:
-            sys.exit(1)
-
-        # Update the translatable strings for this branch
-        zanata_args = [
-            '/usr/bin/zanata-cli', '-B', '-e', 'push',
-            '--url', ctx.parent.obj['zanata_url'],
-            '--project', ctx.parent.obj['zanata_project'],
-            '--project-type', 'gettext',
-            '--project-version', ctx.parent.obj['branch'],
-            '--src-dir', tdir,
-            '--user-config', ctx.parent.obj['zanata_user_config']
-        ]
-        result = subprocess.run(zanata_args, capture_output=True)
-        if result.returncode or ctx.parent.obj['debug']:
-            print(result.stderr.decode('utf-8'))
-            print(result.stdout.decode('utf-8'))
-        if result.returncode:
-            sys.exit(2)
-
-        print("Uploaded translatable strings for %s to Zanata" % (
-            ctx.parent.obj['branch']))
-
-    else:
-        # Move the temporary path to the current directory
-        shutil.move(potfile, po_basename)
-        print("Wrote extracted strings for %s to %s" % (ctx.obj['branch'],
-                                                        po_basename))
-    # Clean up the temporary directory
-    shutil.rmtree(tdir)
+    pofile.write_po(pot_file, catalog, sort_by_file=True)
+
+    print("Wrote extracted strings for %s to %s" % (ctx.obj['branch'],
+                                                    pot_file.name))
 
 
 ##############################################################################
 # `mmdzanata generate_metadata`                                              #
 ##############################################################################
 
 @cli.command()
+@click.option('-d', '--pofile-dir',
+              default='.',
+              help="Path to a directory containing portable object (.po) "
+                   "translation files",
+              type=click.Path(exists=True, dir_okay=True, resolve_path=True,
+                              readable=True))
+@click.option('-y', '--yaml-file',
+              default='fedora-modularity-translations.yaml',
+              type=click.File(mode='wb', atomic=True, lazy=True),
+              show_default=True,
+              metavar="<PATH>",
+              help="Path to the YAML file to hold the translated strings in "
+                   "modulemd-translations format.")
 @click.pass_context
-def generate_metadata(ctx):
+def generate_metadata(ctx, pofile_dir, yaml_file):
     """
     Generate modulemd-translations YAML.
 
     :return: 0 on successful creation of modulemd-translation,
     nonzero on failure.
     """
 
-    zanata_rest_url = "%s/rest" % ctx.parent.obj['zanata_url']
+    # Process all .po files in the provided directory
+    translation_files = [f for f in os.listdir(pofile_dir) if
+                         os.path.isfile((os.path.join(pofile_dir, f))) and
+                         f.endswith(".po")]
+    translations = get_modulemd_translations(translation_files,
+                                             debug=ctx.parent.obj['debug'])
+
+    yaml_file.write(Modulemd.dumps(sorted(translations)).encode('utf-8'))
+
+    print("Wrote modulemd-translations YAML to %s" % yaml_file.name)
+
+
+@cli.command()
+
+@click.pass_context
+def experiment(ctx, pofile_dir):
+    """
+    Experiment with Babel
+    """
 
-    translations = mmdzanata.get_modulemd_translations(
-        zanata_rest_url,
-        ctx.parent.obj['zanata_project'],
-        ctx.parent.obj['branch'],
-        ctx.parent.obj['zanata_translation_document'],
-        ctx.parent.obj['debug']
-    )
+    # Process all .po files in the provided directory
+    translation_files = [f for f in os.listdir(pofile_dir) if
+                         os.path.isfile((os.path.join(pofile_dir, f))) and
+                         f.endswith(".po")]
 
-    translation_file = "%s-%s.yaml" % (
-        ctx.parent.obj['zanata_translation_document'],
-        ctx.parent.obj['branch'])
+    catalogs = dict()
+    for f in translation_files:
+        with open(f, 'r') as infile:
+            catalog = pofile.read_po(infile)
+            catalogs[catalog.locale_identifier] = catalog
 
-    Modulemd.dump(sorted(translations), translation_file)
+    translations = get_modulemd_translations_from_catalog_dict(
+        catalogs)
 
-    print("Wrote modulemd-translations YAML to %s" % translation_file)
+    if ctx.parent.obj['debug']:
+        for translation in translations:
+            print(translation.dumps())
 
 
 if __name__ == "__main__":