# -*- coding: utf-8 -*- #
# Copyright 2017 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""gcloud CLI tree generators for non-gcloud CLIs.

A CLI tree for a supported command is generated by using the root command plus
`help` or `--help` arguments to do a DFS traversal. Each node is generated
from a man-ish style runtime document.

Supported CLI commands have their own runtime help document quirks, so each is
handled by an ad-hoc parser. The parsers rely on consistency within commands
and between command releases.

The CLI tree for an unsupported command is generated from the output of
`man the-command` and contains only the command root node.
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals

import abc
import json
import os
import re
import subprocess
import sys
import tarfile
import textwrap

from googlecloudsdk.calliope import cli_tree
from googlecloudsdk.command_lib.static_completion import generate as generate_static
from googlecloudsdk.command_lib.static_completion import lookup
from googlecloudsdk.core import exceptions
from googlecloudsdk.core import http
from googlecloudsdk.core import log
from googlecloudsdk.core.console import progress_tracker
from googlecloudsdk.core.resource import resource_printer
from googlecloudsdk.core.util import encoding
from googlecloudsdk.core.util import files
from googlecloudsdk.core.util import text as text_utils

import six
from six.moves import range


class Error(exceptions.Error):
  """Exceptions for this module."""


class NoCliTreeForCommand(Error):
  """Command does not have a CLI tree."""


class NoCliTreeGeneratorForCommand(Error):
  """Command does not have a CLI tree generator."""


class NoManPageTextForCommand(Error):
  """Could not get man page text for command."""


# TODO(b/69033748): disable until issues resolved
def _DisableLongRunningCliTreeGeneration(command):
  """Returns True if long running CLI tree generation is disabled."""
  # Only a few command generators are long running.
  if command not in ('bq', 'gsutil', 'kubectl'):
    return False
  # Only generate these CLI trees on the fly for explicit requests.
  # It can take ~1minute, not good, especially at the interactive prompt.
  if 'update-cli-trees' in sys.argv or '--update-cli-trees' in sys.argv:
    return False
  # It's a long running generator, not explicitly requested -- disable.
  return True


def _NormalizeSpace(text):
  """Returns text dedented and multiple non-indent spaces replaced by one."""
  return re.sub('([^ ])   *', r'\1 ', textwrap.dedent(text)).strip('\n')


def _Flag(name, description='', value=None, default=None, type_='string',
          category='', is_global=False, is_required=False, nargs=None):
  """Initializes and returns a flag dict node."""
  return {
      cli_tree.LOOKUP_ATTR: {},
      cli_tree.LOOKUP_CATEGORY: category,
      cli_tree.LOOKUP_DEFAULT: default,
      cli_tree.LOOKUP_DESCRIPTION: _NormalizeSpace(description),
      cli_tree.LOOKUP_GROUP: '',
      cli_tree.LOOKUP_IS_GLOBAL: is_global,
      cli_tree.LOOKUP_IS_HIDDEN: False,
      cli_tree.LOOKUP_IS_REQUIRED: is_required,
      cli_tree.LOOKUP_NAME: name,
      cli_tree.LOOKUP_NARGS: nargs or ('0' if type_ == 'bool' else '1'),
      cli_tree.LOOKUP_VALUE: value,
      cli_tree.LOOKUP_TYPE: type_,
  }


def _Positional(name, description='', default=None, nargs='0'):
  """Initializes and returns a positional dict node."""
  return {
      cli_tree.LOOKUP_DEFAULT: default,
      cli_tree.LOOKUP_DESCRIPTION: _NormalizeSpace(description),
      cli_tree.LOOKUP_NAME: name,
      cli_tree.LOOKUP_NARGS: nargs,
  }


def _Command(path):
  """Initializes and returns a command/group dict node."""
  return {
      cli_tree.LOOKUP_CAPSULE: '',
      cli_tree.LOOKUP_COMMANDS: {},
      cli_tree.LOOKUP_FLAGS: {},
      cli_tree.LOOKUP_GROUPS: {},
      cli_tree.LOOKUP_IS_GROUP: False,
      cli_tree.LOOKUP_IS_HIDDEN: False,
      cli_tree.LOOKUP_PATH: path,
      cli_tree.LOOKUP_POSITIONALS: [],
      cli_tree.LOOKUP_RELEASE: 'GA',
      cli_tree.LOOKUP_SECTIONS: {},
  }


def _GetDirectories(directory=None, warn_on_exceptions=False):
  """Returns the list of directories to search for CLI trees.

  Args:
    directory: The directory containing the CLI tree JSON files. If None
      then the default installation and config directories are used.
    warn_on_exceptions: Emits warning messages in lieu of exceptions.
  """
  # Initialize the list of directories to search for CLI tree files. The default
  # CLI tree is only searched for and generated in directories[0]. Other
  # existing trees are updated in the directory in which they were found. New
  # trees are generated in directories[-1].
  directories = []
  if directory:
    directories.append(directory)
  else:
    try:
      directories.append(cli_tree.CliTreeDir())
    except cli_tree.SdkRootNotFoundError as e:
      if not warn_on_exceptions:
        raise
      log.warning(six.text_type(e))
    directories.append(cli_tree.CliTreeConfigDir())
  return directories


class CliTreeGenerator(six.with_metaclass(abc.ABCMeta, object)):
  """Base CLI tree generator."""

  _FAILURES = None

  @classmethod
  def MemoizeFailures(cls, enable):
    """Memoizes failed attempts and doesn't repeat them if enable is True."""
    cls._FAILURES = set() if enable else None

  @classmethod
  def AlreadyFailed(cls, command):
    """Returns True if man page request for command already failed."""
    return command in cls._FAILURES if cls._FAILURES else False

  @classmethod
  def AddFailure(cls, command):
    """Add command to the set of failed man generations."""
    if cls._FAILURES is not None:
      cls._FAILURES.add(command)

  def __init__(self, command, command_name=None, tarball=None):
    self.command = command
    self.command_name = command_name or command
    self._cli_version = None  # For memoizing GetVersion()
    self._tarball = tarball

  def Run(self, cmd):
    """Runs cmd and returns the output as a string."""
    return encoding.Decode(subprocess.check_output([self.command] + cmd[1:]))

  def GetVersion(self):
    """Returns the CLI_VERSION string."""
    if not self._cli_version:
      try:
        self._cli_version = self.Run([self.command, 'version']).split()[-1]
      except:  # pylint: disable=bare-except
        self._cli_version = cli_tree.CLI_VERSION_UNKNOWN
    return self._cli_version

  @abc.abstractmethod
  def Generate(self):
    """Generates and returns the CLI tree dict."""
    return None

  def FindTreeFile(self, directories):
    """Returns (path,f) open for read for the first CLI tree in directories."""
    for directory in directories or _GetDirectories(warn_on_exceptions=True):
      path = os.path.join(directory or '.', self.command_name) + '.json'
      try:
        return path, files.FileReader(path)
      except files.Error:
        pass
    return path, None

  def IsUpToDate(self, tree, verbose=False):
    """Returns a bool tuple (readonly, up_to_date)."""

    actual_cli_version = tree.get(cli_tree.LOOKUP_CLI_VERSION)
    readonly = actual_cli_version == cli_tree.CLI_VERSION_READONLY

    # Check the schema version.
    actual_tree_version = tree.get(cli_tree.LOOKUP_VERSION)
    if actual_tree_version != cli_tree.VERSION:
      return readonly, False

    # Check the CLI version.
    expected_cli_version = self.GetVersion()
    if readonly:
      # READONLY trees are always up to date.
      pass
    elif expected_cli_version == cli_tree.CLI_VERSION_UNKNOWN:
      # Don't know how to regenerate but we have one in hand -- accept it.
      pass
    elif actual_cli_version != expected_cli_version:
      return readonly, False

    # Schema and CLI versions are up to date.
    if verbose:
      log.status.Print('[{}] CLI tree version [{}] is up to date.'.format(
          self.command, actual_cli_version))
    return readonly, True

  def LoadOrGenerate(self, directories=None, force=False, generate=True,
                     ignore_out_of_date=False, tarball=False, verbose=False,
                     warn_on_exceptions=False):
    """Loads the CLI tree or generates it if necessary, and returns the tree."""
    f = None
    try:
      path, f = self.FindTreeFile(directories)
      if not f:
        # TODO(b/69033748): disable until issues resolved
        if _DisableLongRunningCliTreeGeneration(self.command):
          return None
      else:
        up_to_date = False
        try:
          tree = json.load(f)
        except ValueError:
          # Corrupt JSON -- could have been interrupted.
          tree = None
        if tree:
          readonly, up_to_date = self.IsUpToDate(tree, verbose=verbose)
          if readonly:
            return tree
          elif up_to_date:
            if not force:
              return tree
          elif ignore_out_of_date:
            return None
    finally:
      if f:
        f.close()

    def _Generate():
      """Helper that generates a CLI tree and writes it to a JSON file."""
      tree = self.Generate()
      if tree:
        try:
          f = files.FileWriter(path)
        except files.Error as e:
          # CLI data config dir may not be initialized yet.
          directory, _ = os.path.split(path)
          try:
            files.MakeDir(directory)
            f = files.FileWriter(path)
          except files.Error:
            if not warn_on_exceptions:
              raise
            log.warning(six.text_type(e))
            return None
        with f:
          resource_printer.Print(tree, print_format='json', out=f)
      return tree

    # At this point:
    #   (1) the tree is not found or is out of date
    #   (2) the tree is not readonly
    #   (3) we have a generator for the tree

    if not generate:
      raise NoCliTreeForCommand('No CLI tree for [{}].'.format(self.command))
    if not verbose:
      return _Generate()
    with progress_tracker.ProgressTracker(
        '{} the [{}] CLI tree'.format(
            'Updating' if f else 'Generating', self.command)):
      return _Generate()


class _BqCollector(object):
  """bq help document section collector."""

  def __init__(self, text):
    self.text = text.split('\n')
    self.heading = 'DESCRIPTION'
    self.lookahead = None
    self.ignore_trailer = False

  def Collect(self, strip_headings=False):
    """Returns the heading and content lines from text."""
    content = []
    if self.lookahead:
      if not strip_headings:
        content.append(self.lookahead)
      self.lookahead = None
    heading = self.heading
    self.heading = None
    while self.text:
      line = self.text.pop(0)
      if line.startswith(' ') or not strip_headings and not self.ignore_trailer:
        content.append(line.rstrip())
    while content and not content[0]:
      content.pop(0)
    while content and not content[-1]:
      content.pop()
    self.ignore_trailer = True
    return heading, content


class BqCliTreeGenerator(CliTreeGenerator):
  """bq CLI tree generator."""

  def Run(self, cmd):
    """Runs cmd and returns the output as a string."""
    try:
      output = subprocess.check_output([self.command] + cmd[1:])
    except subprocess.CalledProcessError as e:
      # bq exit code is 1 for help and --help. How do you know if help failed?
      if e.returncode != 1:
        raise
      output = e.output
    return encoding.Decode(output).replace('bq.py', 'bq')

  def AddFlags(self, command, content, is_global=False):
    """Adds flags in content lines to command."""
    while content:
      line = content.pop(0)
      name, description = line.strip().split(':', 1)
      paragraph = [description.strip()]
      default = ''
      while content and not content[0].startswith('  --'):
        line = content.pop(0).strip()
        if line.startswith('(default: '):
          default = line[10:-1]
        else:
          paragraph.append(line)
      description = ' '.join(paragraph).strip()
      if name.startswith('--[no]'):
        name = '--' + name[6:]
        type_ = 'bool'
        value = ''
      else:
        value = 'VALUE'
        type_ = 'string'
      command[cli_tree.LOOKUP_FLAGS][name] = _Flag(
          name=name,
          description=description,
          type_=type_,
          value=value,
          default=default,
          is_required=False,
          is_global=is_global,
      )

  def SubTree(self, path):
    """Generates and returns the CLI subtree rooted at path."""
    command = _Command(path)
    command[cli_tree.LOOKUP_IS_GROUP] = True
    text = self.Run([self.command, 'help'] + path[1:])

    # `bq help` lists help for all commands. Command flags are "defined"
    # by example. We don't attempt to suss that out.
    content = text.split('\n')
    while content:
      line = content.pop(0)
      if not line or not line[0].islower():
        continue
      name, text = line.split(' ', 1)
      description = [text.strip()]
      examples = []
      arguments = []
      paragraph = description
      while content and (not content[0] or not content[0][0].islower()):
        line = content.pop(0).strip()
        if line == 'Arguments:':
          paragraph = arguments
        elif line == 'Examples:':
          paragraph = examples
        else:
          paragraph.append(line)
      subcommand = _Command(path + [name])
      command[cli_tree.LOOKUP_COMMANDS][name] = subcommand
      if description:
        subcommand[cli_tree.LOOKUP_SECTIONS]['DESCRIPTION'] = '\n'.join(
            description)
      if examples:
        subcommand[cli_tree.LOOKUP_SECTIONS]['EXAMPLES'] = '\n'.join(
            examples)

    return command

  def Generate(self):
    """Generates and returns the CLI tree rooted at self.command."""

    # Construct the tree minus the global flags.
    tree = self.SubTree([self.command_name])

    # Add the global flags to the root.
    text = self.Run([self.command, '--help'])
    collector = _BqCollector(text)
    _, content = collector.Collect(strip_headings=True)
    self.AddFlags(tree, content, is_global=True)

    # Finally add the version stamps.
    tree[cli_tree.LOOKUP_CLI_VERSION] = self.GetVersion()
    tree[cli_tree.LOOKUP_VERSION] = cli_tree.VERSION

    return tree


class _GsutilCollector(object):
  """gsutil help document section collector."""

  UNKNOWN, ROOT, MAN, TOPIC = list(range(4))

  def __init__(self, text):
    self.text = text.split('\n')
    self.heading = 'CAPSULE'
    self.page_type = self.UNKNOWN

  def Collect(self, strip_headings=False):
    """Returns the heading and content lines from text."""
    content = []
    heading = self.heading
    self.heading = None
    while self.text:
      line = self.text.pop(0)
      if self.page_type == self.UNKNOWN:
        # The first heading distinguishes the document page type.
        if line.startswith('Usage:'):
          self.page_type = self.ROOT
          continue
        elif line == 'NAME':
          self.page_type = self.MAN
          heading = 'CAPSULE'
          continue
        elif not line.startswith(' '):
          continue
      elif self.page_type == self.ROOT:
        # The root help page.
        if line == 'Available commands:':
          heading = 'COMMANDS'
          continue
        elif line == 'Additional help topics:':
          self.heading = 'TOPICS'
          break
        elif not line.startswith(' '):
          continue
      elif self.page_type == self.MAN:
        # A command/subcommand man style page.
        if line == 'OVERVIEW':
          self.page_type = self.TOPIC
          self.heading = 'DESCRIPTION'
          break
        elif line == 'SYNOPSIS':
          self.heading = line
          break
        elif line.endswith('OPTIONS'):
          self.heading = 'FLAGS'
          break
        elif line and line[0].isupper():
          self.heading = line.split(' ', 1)[-1]
          break
      elif self.page_type == self.TOPIC:
        # A topic man style page.
        if line and line[0].isupper():
          self.heading = line
          break
      if line.startswith(' ') or not strip_headings:
        content.append(line.rstrip())
    while content and not content[0]:
      content.pop(0)
    while content and not content[-1]:
      content.pop()
    return heading, content


class GsutilCliTreeGenerator(CliTreeGenerator):
  """gsutil CLI tree generator."""

  def __init__(self, *args, **kwargs):
    super(GsutilCliTreeGenerator, self).__init__(*args, **kwargs)
    self.topics = []

  def Run(self, cmd):
    """Runs the command in cmd and returns the output as a string."""
    try:
      if self._tarball:
        # package time invocation requires explicit #! override
        cmd = [sys.executable, self.command] + cmd[1:]
      output = subprocess.check_output(cmd)
    except subprocess.CalledProcessError as e:
      # gsutil exit code is 1 for --help depending on the context.
      if e.returncode != 1:
        raise
      output = e.output
    return encoding.Decode(output)

  def AddFlags(self, command, content, is_global=False):
    """Adds flags in content lines to command."""

    def _Add(name, description):
      value = ''
      type_ = 'bool'
      default = ''
      command[cli_tree.LOOKUP_FLAGS][name] = _Flag(
          name=name,
          description=description,
          type_=type_,
          value=value,
          default=default,
          is_required=False,
          is_global=is_global,
      )

    parse = re.compile(' *((-[^ ]*,)* *(-[^ ]*) *)(.*)')
    name = None
    description = []
    for line in content:
      if line.startswith('  -'):
        if name:
          _Add(name, '\n'.join(description))
        match = parse.match(line)
        name = match.group(3)
        description = [match.group(4).rstrip()]
      elif len(line) > 16:
        description.append(line[16:].rstrip())
    if name:
      _Add(name, '\n'.join(description))

  def SubTree(self, path):
    """Generates and returns the CLI subtree rooted at path."""
    command = _Command(path)
    is_help_command = len(path) > 1 and path[1] == 'help'
    if is_help_command:
      cmd = path
    else:
      cmd = path + ['--help']
    text = self.Run(cmd)
    collector = _GsutilCollector(text)

    while True:
      heading, content = collector.Collect()
      if not heading:
        break
      elif heading == 'CAPSULE':
        if content:
          command[cli_tree.LOOKUP_CAPSULE] = content[0].split('-', 1)[1].strip()
      elif heading == 'COMMANDS':
        if is_help_command:
          continue
        for line in content:
          try:
            name = line.split()[0]
          except IndexError:
            continue
          if name == 'update':
            continue
          command[cli_tree.LOOKUP_IS_GROUP] = True
          command[cli_tree.LOOKUP_COMMANDS][name] = self.SubTree(path + [name])
      elif heading == 'FLAGS':
        self.AddFlags(command, content)
      elif heading == 'SYNOPSIS':
        commands = []
        for line in content:
          if not line:
            break
          cmd = line.split()
          if len(cmd) <= len(path):
            continue
          if cmd[:len(path)] == path:
            name = cmd[len(path)]
            if name[0].islower() and name not in ('off', 'on', 'false', 'true'):
              commands.append(name)
        if len(commands) > 1:
          command[cli_tree.LOOKUP_IS_GROUP] = True
          for name in commands:
            command[cli_tree.LOOKUP_COMMANDS][name] = self.SubTree(
                path + [name])
      elif heading == 'TOPICS':
        for line in content:
          try:
            self.topics.append(line.split()[0])
          except IndexError:
            continue
      elif heading.isupper():
        if heading.lower() == path[-1]:
          heading = 'DESCRIPTION'
        command[cli_tree.LOOKUP_SECTIONS][heading] = '\n'.join(
            [line[2:] for line in content])

    return command

  def Generate(self):
    """Generates and returns the CLI tree rooted at self.command."""
    tree = self.SubTree([self.command_name])

    # Add the global flags to the root.
    text = self.Run([self.command, 'help', 'options'])
    collector = _GsutilCollector(text)
    while True:
      heading, content = collector.Collect()
      if not heading:
        break
      if heading == 'FLAGS':
        self.AddFlags(tree, content, is_global=True)

    # Add the help topics.
    help_command = tree[cli_tree.LOOKUP_COMMANDS]['help']
    help_command[cli_tree.LOOKUP_IS_GROUP] = True
    for topic in self.topics:
      help_command[cli_tree.LOOKUP_COMMANDS][topic] = self.SubTree(
          help_command[cli_tree.LOOKUP_PATH] + [topic])

    # Finally add the version stamps.
    tree[cli_tree.LOOKUP_CLI_VERSION] = self.GetVersion()
    tree[cli_tree.LOOKUP_VERSION] = cli_tree.VERSION

    return tree


class _KubectlCollector(object):
  """Kubectl help document section collector."""

  def __init__(self, text):
    self.text = text.split('\n')
    self.heading = 'DESCRIPTION'
    self.lookahead = None
    self.ignore_trailer = False

  def Collect(self, strip_headings=False):
    """Returns the heading and content lines from text."""
    content = []
    if self.lookahead:
      if not strip_headings:
        content.append(self.lookahead)
      self.lookahead = None
    heading = self.heading
    self.heading = None
    while self.text:
      line = self.text.pop(0)
      usage = 'Usage:'
      if line.startswith(usage):
        line = line[len(usage):].strip()
        if line:
          self.lookahead = line
        self.heading = 'USAGE'
        break
      if line.endswith(':'):
        if 'Commands' in line:
          self.heading = 'COMMANDS'
          break
        if 'Examples' in line:
          self.heading = 'EXAMPLES'
          break
        if 'Options' in line:
          self.heading = 'FLAGS'
          break
      if line.startswith(' ') or not strip_headings and not self.ignore_trailer:
        content.append(line.rstrip())
    while content and not content[0]:
      content.pop(0)
    while content and not content[-1]:
      content.pop()
    self.ignore_trailer = True
    return heading, content


class KubectlCliTreeGenerator(CliTreeGenerator):
  """kubectl CLI tree generator."""

  def AddFlags(self, command, content, is_global=False):
    """Adds flags in content lines to command."""
    for line in content:
      flags, description = line.strip().split(':', 1)
      flag = flags.split(', ')[-1]
      name, value = flag.split('=')
      if value in ('true', 'false'):
        value = ''
        type_ = 'bool'
      else:
        value = 'VALUE'
        type_ = 'string'
      default = ''
      command[cli_tree.LOOKUP_FLAGS][name] = _Flag(
          name=name,
          description=description,
          type_=type_,
          value=value,
          default=default,
          is_required=False,
          is_global=is_global,
      )

  def SubTree(self, path):
    """Generates and returns the CLI subtree rooted at path."""
    command = _Command(path)
    text = self.Run(path + ['--help'])
    collector = _KubectlCollector(text)

    while True:
      heading, content = collector.Collect()
      if not heading:
        break
      elif heading == 'COMMANDS':
        for line in content:
          try:
            name = line.split()[0]
          except IndexError:
            continue
          command[cli_tree.LOOKUP_IS_GROUP] = True
          command[cli_tree.LOOKUP_COMMANDS][name] = self.SubTree(path + [name])
      elif heading in ('DESCRIPTION', 'EXAMPLES'):
        command[cli_tree.LOOKUP_SECTIONS][heading] = '\n'.join(content)
      elif heading == 'FLAGS':
        self.AddFlags(command, content)
    return command

  def GetVersion(self):
    """Returns the CLI_VERSION string."""
    if not self._cli_version:
      try:
        verbose_version = self.Run([self.command, 'version', '--client'])
        match = re.search('GitVersion:"([^"]*)"', verbose_version)
        self._cli_version = match.group(1)
      except:  # pylint: disable=bare-except
        self._cli_version = cli_tree.CLI_VERSION_UNKNOWN
    return self._cli_version

  def Generate(self):
    """Generates and returns the CLI tree rooted at self.command."""

    # Construct the tree minus the global flags.
    tree = self.SubTree([self.command_name])

    # Add the global flags to the root.
    text = self.Run([self.command, 'options'])
    collector = _KubectlCollector(text)
    _, content = collector.Collect(strip_headings=True)
    content.append('  --help=true: List detailed command help.')
    self.AddFlags(tree, content, is_global=True)

    # Finally add the version stamps.
    tree[cli_tree.LOOKUP_CLI_VERSION] = self.GetVersion()
    tree[cli_tree.LOOKUP_VERSION] = cli_tree.VERSION

    return tree


class _ManPageCollector(six.with_metaclass(abc.ABCMeta, object)):
  """man page help document section collector base class.

  Attributes:
    command: The man page command name.
    content_indent: A string of space characters representing the indent of
      the first line of content for any section.
    heading: The heading for the next call to Collect().
    text: The list of man page lines.
    version: The collector CLI_VERSION string.
  """

  def __init__(self, command):
    self.command = command
    self.content_indent = None
    self.heading = None
    self.text = self.GetManPageText().split('\n')

  @classmethod
  @abc.abstractmethod
  def GetVersion(cls):
    """Returns the CLI_VERSION string."""
    return None

  @abc.abstractmethod
  def _GetRawManPageText(self):
    """Returns the raw man page text."""
    return None

  @abc.abstractmethod
  def GetManPageText(self):
    """Returns the preprocessed man page text."""
    return None

  def Collect(self):
    """Returns the heading and content lines from text."""
    content = []
    heading = self.heading
    self.heading = None
    while self.text:
      line = self.text.pop(0)
      if not heading:
        # No NAME no man page.
        if line == 'NAME':
          heading = line
        continue
      elif not line:
        pass
      elif line[0] == ' ':
        if not self.content_indent:
          self.content_indent = re.sub('[^ ].*', '', line)
        if len(line) > len(self.content_indent):
          indented_char = line[len(self.content_indent)]
          if not line.startswith(self.content_indent):
            # Subsection heading or category.
            line = '### ' + line.strip()
          elif heading == 'DESCRIPTION' and indented_char == '-':
            # Some man pages, like GNU ls(1), inline flags in DESCRIPTION.
            self.text.insert(0, line)
            self.heading = 'FLAGS'
            break
          elif heading == 'FLAGS' and indented_char not in (' ', '-'):
            self.text.insert(0, line)
            self.heading = 'DESCRIPTION'
            break
      elif line in ('SYNOPSIS', 'DESCRIPTION', 'EXIT STATUS', 'SEE ALSO'):
        self.heading = line
        break
      elif 'FLAGS' in line or 'OPTIONS' in line:
        self.heading = 'FLAGS'
        break
      elif line and line[0].isupper():
        self.heading = line.split(' ', 1)[-1]
        break
      content.append(line.rstrip())
    while content and not content[0]:
      content.pop(0)
    while content and not content[-1]:
      content.pop()
    return heading, content


class _ManCommandCollector(_ManPageCollector):
  """man command help document section collector."""

  _CLI_VERSION = 'man-0.1'

  @classmethod
  def GetVersion(cls):
    return cls._CLI_VERSION

  def _GetRawManPageText(self):
    """Returns the raw man page text."""
    try:
      with files.FileWriter(os.devnull) as f:
        return encoding.Decode(subprocess.check_output(['man', self.command],
                                                       stderr=f))
    except (OSError, subprocess.CalledProcessError):
      raise NoManPageTextForCommand(
          'Cannot get man(1) command man page text for [{}].'.format(
              self.command))

  def GetManPageText(self):
    """Returns the preprocessed man page text."""
    text = self._GetRawManPageText()
    return re.sub(
        '.\b', '', re.sub(
            '(\u2010|\\u2010)\n *', '', text))


class _ManUrlCollector(_ManPageCollector):
  """man URL help document section collector."""

  _CLI_VERSION = 'man7.org-0.1'

  @classmethod
  def GetVersion(cls):
    return cls._CLI_VERSION

  def _GetRawManPageText(self):
    """Returns the raw man page text."""
    url = 'http://man7.org/linux/man-pages/man1/{}.1.html'.format(self.command)
    response, content = http.HttpClient().request(url)
    if response.status != 200:
      raise NoManPageTextForCommand(
          'Cannot get URL man page text for [{}].'.format(self.command))
    return content.decode('utf-8')

  def GetManPageText(self):
    """Returns the text man page for self.command from a URL."""
    text = self._GetRawManPageText()

    # A little sed(1) to clean up the html header/trailer and anchors.
    # A mispplaced .* or (...) group match could affect timing. Keep track
    # of that if you change any of the pattern,replacement tuples.
    for pattern, replacement in (
        ('<span class="footline">.*', ''),
        ('<h2><a id="([^"]*)"[^\n]*\n', '\\1\n'),
        ('<b>( +)', '\\1*'),
        ('( +)</b>', '*\\1'),
        ('<i>( +)', '\\1_'),
        ('( +)</i>', '_\\1'),
        ('</?b>', '*'),
        ('</?i>', '_'),
        ('</pre>', ''),
        ('<a href="([^"]*)">([^\n]*)</a>', '[\\1](\\2)'),
        ('&amp;', '\\&'),
        ('&gt;', '>'),
        ('&lt;', '<'),
        ('&#39;', "'"),
    ):
      text = re.sub(pattern, replacement, text, flags=re.DOTALL)

    # ... and some non-regular edits to finish up.
    lines = []
    top = 'NAME'
    flags = False
    paragraph = False
    for line in text.split('\n'):
      if top and line == 'NAME':
        # Ignore all lines before the NAME section.
        top = None
        lines = []
      if line.startswith('       *-'):  # NOTICE: NOT a regex!
        # Drop font embellishment markdown from flag definition list lines.
        flags = True
        if paragraph:
          # Blank line was really a paragraph.
          paragraph = False
          lines.append('')
        if '  ' in line[7:]:
          head, tail = line[7:].split('  ', 1)
          head = re.sub('\\*', '', head)
          line = '  '.join(['     ', head, tail])
        else:
          line = re.sub('\\*', '', line)
      elif flags:
        if not line:
          paragraph = True
          continue
        elif not line.startswith('       '):
          flags = False
          paragraph = False
        elif paragraph:
          if not line[0].lower():
            # A real paragraph
            lines.append('+')
          paragraph = False
      lines.append(line)
    return '\n'.join(lines)


class ManPageCliTreeGenerator(CliTreeGenerator):
  """man page CLI tree generator."""

  @classmethod
  def _GetManPageCollectorType(cls):
    """Returns the man page collector type."""
    if files.FindExecutableOnPath('man'):
      return _ManCommandCollector
    return _ManUrlCollector

  def __init__(self, command):
    super(ManPageCliTreeGenerator, self).__init__(command)
    self.collector_type = self._GetManPageCollectorType()

  def GetVersion(self):
    """Returns the CLI_VERSION string."""
    if not self.collector_type:
      return cli_tree.CLI_VERSION_UNKNOWN
    return self.collector_type.GetVersion()

  def AddFlags(self, command, content, is_global=False):
    """Adds flags in content lines to command."""

    def _NameTypeValueNargs(name, type_=None, value=None, nargs=None):
      """Returns the (name, type, value-metavar, nargs) for flag name."""
      if name.startswith('--'):
        # --foo
        if '=' in name:
          name, value = name.split('=', 1)
          if name[-1] == '[':
            name = name[:-1]
            if value.endswith(']'):
              value = value[:-1]
            nargs = '?'
          else:
            nargs = '1'
          type_ = 'string'
      elif len(name) > 2:
        # -b VALUE
        value = name[2:]
        if value[0].isspace():
          value = value[1:]
        if value.startswith('['):
          value = value[1:]
          if value.endswith(']'):
            value = value[:-1]
          nargs = '?'
        else:
          nargs = '1'
        name = name[:2]
        type_ = 'string'
      if type_ is None or value is None or nargs is None:
        type_ = 'bool'
        value = ''
        nargs = '0'
      return (name, type_, value, nargs)

    def _Add(name, description, category, type_, value, nargs):
      """Adds a flag."""
      name, type_, value, nargs = _NameTypeValueNargs(name, type_, value, nargs)
      default = ''
      command[cli_tree.LOOKUP_FLAGS][name] = _Flag(
          name=name,
          description='\n'.join(description),
          type_=type_,
          value=value,
          nargs=nargs,
          category=category,
          default=default,
          is_required=False,
          is_global=is_global,
      )

    def _AddNames(names, description, category):
      """Add a flag name list."""
      if names:
        _, type_, value, nargs = _NameTypeValueNargs(names[-1])
        for name in names:
          _Add(name, description, category, type_, value, nargs)

    names = []
    description = []
    category = ''
    for line in content:
      if line.lstrip().startswith('-'):
        _AddNames(names, description, category)
        line = line.lstrip()
        names = line.strip().replace(', -', ', --').split(', -')
        if ' ' in names[-1]:
          names[-1], text = names[-1].split(' ', 1)
          description = [text.strip()]
        else:
          description = []
      elif line.startswith('### '):
        category = line[4:]
      else:
        description.append(line)
    _AddNames(names, description, category)

  def _Generate(self, path, collector):
    """Generates and returns the CLI subtree rooted at path."""
    command = _Command(path)

    while True:
      heading, content = collector.Collect()
      if not heading:
        break
      elif heading == 'NAME':
        if content:
          command[cli_tree.LOOKUP_CAPSULE] = re.sub(
              '.* -+  *', '', content[0]).strip()
      elif heading == 'FLAGS':
        self.AddFlags(command, content)
      elif heading not in ('BUGS', 'COLOPHON', 'COMPATIBILITY', 'HISTORY',
                           'STANDARDS', 'SYNOPSIS'):
        blocks = []
        begin = 0
        end = 0
        while end < len(content):
          if content[end].startswith('###'):
            if begin < end:
              blocks.append(_NormalizeSpace('\n'.join(content[begin:end])))
            blocks.append(content[end])
            begin = end + 1
          end += 1
        if begin < end:
          blocks.append(_NormalizeSpace('\n'.join(content[begin:end])))
        text = '\n'.join(blocks)
        if heading in command[cli_tree.LOOKUP_SECTIONS]:
          command[cli_tree.LOOKUP_SECTIONS][heading] += '\n\n' + text
        else:
          command[cli_tree.LOOKUP_SECTIONS][heading] = text
    return command

  def Generate(self):
    """Generates and returns the CLI tree rooted at self.command."""
    if not self.collector_type:
      return None
    collector = self.collector_type(self.command)
    if not collector:
      return None
    tree = self._Generate([self.command], collector)
    if not tree:
      return None

    # Add the version stamps.
    tree[cli_tree.LOOKUP_CLI_VERSION] = self.GetVersion()
    tree[cli_tree.LOOKUP_VERSION] = cli_tree.VERSION

    return tree


GENERATORS = {
    'bq': BqCliTreeGenerator,
    'gsutil': GsutilCliTreeGenerator,
    'kubectl': KubectlCliTreeGenerator,
}


def LoadOrGenerate(command, directories=None, tarball=None, force=False,
                   generate=True, ignore_out_of_date=False, verbose=False,
                   warn_on_exceptions=False, allow_extensions=False):
  """Returns the CLI tree for command, generating it if it does not exist.

  Args:
    command: The CLI root command name.
    directories: The list of directories containing the CLI tree JSON files.
      If None then the default installation directories are used.
    tarball: For packaging CLI trees. --commands specifies one command that is
      a relative path in this tarball. The tarball is extracted to a temporary
      directory and the command path is adjusted to point to the temporary
      directory.
    force: Update all exitsing trees by forcing them to be out of date if True.
    generate: Generate the tree if it is out of date or does not exist.
    ignore_out_of_date: Ignore out of date trees instead of regenerating.
    verbose: Display a status line for up to date CLI trees if True.
    warn_on_exceptions: Emits warning messages in lieu of generator exceptions.
      Used during installation.
    allow_extensions: Whether or not to allow extensions in executable names.

  Returns:
    The CLI tree for command or None if command not found or there is no
      generator.
  """
  command_dir, command_name = os.path.split(command)

  # Handle package time command names.
  if command_name.endswith('_lite'):
    command_name = command_name[:-5]

  # Don't repeat failed attempts.
  if CliTreeGenerator.AlreadyFailed(command_name):
    if verbose:
      log.status.Print('No CLI tree generator for [{}].'.format(command))
    return None

  def _LoadOrGenerate(command, command_dir, command_name):
    """Helper."""

    # The command must exist.
    if (command_dir and not os.path.exists(command) or
        not command_dir
        and not files.FindExecutableOnPath(command_name,
                                           allow_extensions=allow_extensions)):
      if verbose:
        log.status.Print('Command [{}] not found.'.format(command))
      return None

    # Instantiate the appropriate generator.
    try:
      generator = GENERATORS[command_name](
          command, command_name=command_name, tarball=tarball)
    except KeyError:
      generator = ManPageCliTreeGenerator(command_name)
    if not generator:
      return None

    # Load or (re)generate the CLI tree if possible.
    try:
      return generator.LoadOrGenerate(
          directories=directories,
          force=force,
          generate=generate,
          ignore_out_of_date=ignore_out_of_date,
          verbose=verbose,
          warn_on_exceptions=warn_on_exceptions)
    except NoManPageTextForCommand:
      pass

    return None

  if command_name == cli_tree.DEFAULT_CLI_NAME:
    tree = cli_tree.Load()
  elif tarball:
    with files.TemporaryDirectory() as tmp:
      tar = tarfile.open(tarball)
      tar.extractall(tmp)
      command = os.path.join(tmp, command)
      command_dir = os.path.join(tmp, command_dir)
      tree = _LoadOrGenerate(command, command_dir, command_name)
  else:
    tree = _LoadOrGenerate(command, command_dir, command_name)
  if not tree:
    CliTreeGenerator.AddFailure(command_name)
  return tree


def UpdateCliTrees(cli=None, commands=None, directory=None, tarball=None,
                   force=False, verbose=False, warn_on_exceptions=False):
  """(re)generates the CLI trees in directory if non-existent or out of date.

  This function uses the progress tracker because some of the updates can
  take ~minutes.

  Args:
    cli: The default CLI. If not None then the default CLI is also updated.
    commands: Update only the commands in this list.
    directory: The directory containing the CLI tree JSON files. If None
      then the default installation directories are used.
    tarball: For packaging CLI trees. --commands specifies one command that is
      a relative path in this tarball. The tarball is extracted to a temporary
      directory and the command path is adjusted to point to the temporary
      directory.
    force: Update all exitsing trees by forcing them to be out of date if True.
    verbose: Display a status line for up to date CLI trees if True.
    warn_on_exceptions: Emits warning messages in lieu of exceptions. Used
      during installation.

  Raises:
    NoCliTreeGeneratorForCommand: A command in commands is not supported
      (doesn't have a generator).
  """
  directories = _GetDirectories(
      directory=directory, warn_on_exceptions=warn_on_exceptions)
  if not commands:
    commands = set([cli_tree.DEFAULT_CLI_NAME] + list(GENERATORS.keys()))

  failed = []
  for command in sorted(commands):
    if command != cli_tree.DEFAULT_CLI_NAME:
      tree = LoadOrGenerate(command,
                            directories=directories,
                            tarball=tarball,
                            force=force,
                            verbose=verbose,
                            warn_on_exceptions=warn_on_exceptions)
      if not tree:
        failed.append(command)
    elif cli:

      def _Mtime(path):
        try:
          return os.path.getmtime(path)
        except OSError:
          return 0

      # Update the CLI tree.
      cli_tree_path = cli_tree.CliTreePath(directory=directories[0])
      cli_tree.Load(cli=cli, path=cli_tree_path, force=force, verbose=verbose)

      # Update the static completion CLI tree if older than the CLI tree. To
      # keep static completion startup lightweight we don't track the release
      # in the tree data. Using the modify time is a minimal sanity check.
      completion_tree_path = lookup.CompletionCliTreePath(
          directory=directories[0])
      cli_tree_mtime = _Mtime(cli_tree_path)
      completion_tree_mtime = _Mtime(completion_tree_path)
      if (force or not completion_tree_mtime or
          completion_tree_mtime < cli_tree_mtime):
        files.MakeDir(os.path.dirname(completion_tree_path))
        with files.FileWriter(completion_tree_path) as f:
          generate_static.ListCompletionTree(cli, out=f)
      elif verbose:
        log.status.Print(
            '[{}] static completion CLI tree is up to date.'.format(command))

  if failed:
    message = 'No CLI tree {} for [{}].'.format(
        text_utils.Pluralize(len(failed), 'generator'),
        ', '.join(sorted(failed)))
    if not warn_on_exceptions:
      raise NoCliTreeGeneratorForCommand(message)
    log.warning(message)


def LoadAll(directory=None, ignore_out_of_date=False, root=None,
            warn_on_exceptions=True):
  """Loads all CLI trees in directory and adds them to tree.

  Args:
    directory: The config directory containing the CLI tree modules.
    ignore_out_of_date: Ignore out of date trees instead of regenerating.
    root: dict, The CLI root to update. A new root is created if None.
    warn_on_exceptions: Warn on exceptions instead of raising if True.

  Raises:
    CliTreeVersionError: loaded tree version mismatch
    ImportModuleError: import errors

  Returns:
    The CLI tree.
  """
  # Create the root node if needed.
  if root is None:
    root = cli_tree.Node(description='The CLI tree root.')

  # Load the default CLI if available.
  if cli_tree.DEFAULT_CLI_NAME not in root[cli_tree.LOOKUP_COMMANDS]:
    try:
      root[cli_tree.LOOKUP_COMMANDS][cli_tree.DEFAULT_CLI_NAME] = (
          cli_tree.Load())
    except cli_tree.CliTreeLoadError:
      pass

  # Load extra CLIs by searching directories in order. .json files are treated
  # as CLI modules/data, where the file base name is the name of the CLI root
  # command.
  directories = _GetDirectories(
      directory=directory, warn_on_exceptions=warn_on_exceptions)

  loaded = {cli_tree.DEFAULT_CLI_NAME, '__init__'}  # Already loaded this above.
  for directory in directories:
    if not directory or not os.path.exists(directory):
      continue
    for (dirpath, _, filenames) in os.walk(directory):
      for filename in sorted(filenames):  # For stability across runs.
        command, extension = os.path.splitext(filename)
        if extension != '.json':
          continue
        if command in loaded:
          # Already loaded. Earlier directory hits take precedence.
          continue
        loaded.add(command)
        if command == cli_tree.DEFAULT_CLI_NAME:
          tree = cli_tree.Load(os.path.join(dirpath, filename))
        else:
          tree = LoadOrGenerate(command,
                                directories=[dirpath],
                                ignore_out_of_date=ignore_out_of_date,
                                warn_on_exceptions=warn_on_exceptions)
        if tree:
          root[cli_tree.LOOKUP_COMMANDS][command] = tree
      # Don't search subdirectories.
      break

  return root
