# -*- coding: utf-8 -*-
# Copyright 2011 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Implementation of Unix-like ls command for cloud storage providers."""

from __future__ import absolute_import

import re

from gslib.cloud_api import NotFoundException
from gslib.command import Command
from gslib.command_argument import CommandArgument
from gslib.cs_api_map import ApiSelector
from gslib.exception import CommandException
from gslib.storage_url import ContainsWildcard
from gslib.storage_url import StorageUrlFromString
from gslib.utils.constants import NO_MAX
from gslib.utils.constants import S3_DELETE_MARKER_GUID
from gslib.utils.constants import UTF8
from gslib.utils.ls_helper import ENCRYPTED_FIELDS
from gslib.utils.ls_helper import LsHelper
from gslib.utils.ls_helper import PrintFullInfoAboutObject
from gslib.utils.ls_helper import UNENCRYPTED_FULL_LISTING_FIELDS
from gslib.utils.text_util import InsistAscii
from gslib.utils.translation_helper import AclTranslation
from gslib.utils.translation_helper import LabelTranslation
from gslib.utils.unit_util import MakeHumanReadable

# Regex that assists with converting JSON timestamp to ls-style output.
# This excludes timestamp fractional seconds, for example:
# 2013-07-03 20:32:53.048000+00:00
JSON_TIMESTAMP_RE = re.compile(r'([^\s]*)\s([^\.\+]*).*')

_SYNOPSIS = """
  gsutil ls [-a] [-b] [-d] [-l] [-L] [-r] [-p proj_id] url...
"""

_DETAILED_HELP_TEXT = ("""
<B>SYNOPSIS</B>
""" + _SYNOPSIS + """


<B>LISTING PROVIDERS, BUCKETS, SUBDIRECTORIES, AND OBJECTS</B>
  If you run gsutil ls without URLs, it lists all of the Google Cloud Storage
  buckets under your default project ID:

    gsutil ls

  (For details about projects, see "gsutil help projects" and also the -p
  option in the OPTIONS section below.)

  If you specify one or more provider URLs, gsutil ls will list buckets at
  each listed provider:

    gsutil ls gs://

  If you specify bucket URLs, gsutil ls will list objects at the top level of
  each bucket, along with the names of each subdirectory. For example:

    gsutil ls gs://bucket

  might produce output like:

    gs://bucket/obj1.htm
    gs://bucket/obj2.htm
    gs://bucket/images1/
    gs://bucket/images2/

  The "/" at the end of the last 2 URLs tells you they are subdirectories,
  which you can list using:

    gsutil ls gs://bucket/images*

  If you specify object URLs, gsutil ls will list the specified objects. For
  example:

    gsutil ls gs://bucket/*.txt

  will list all files whose name matches the above wildcard at the top level
  of the bucket.

  See "gsutil help wildcards" for more details on working with wildcards.


<B>DIRECTORY BY DIRECTORY, FLAT, and RECURSIVE LISTINGS</B>
  Listing a bucket or subdirectory (as illustrated near the end of the previous
  section) only shows the objects and names of subdirectories it contains. You
  can list all objects in a bucket by using the -r option. For example:

    gsutil ls -r gs://bucket

  will list the top-level objects and buckets, then the objects and
  buckets under gs://bucket/images1, then those under gs://bucket/images2, etc.

  If you want to see all objects in the bucket in one "flat" listing use the
  recursive ("**") wildcard, like:

    gsutil ls -r gs://bucket/**

  or, for a flat listing of a subdirectory:

    gsutil ls -r gs://bucket/dir/**

  If you want to see only the subdirectory itself, use the -d option:

    gsutil ls -d gs://bucket/dir


<B>LISTING OBJECT DETAILS</B>
  If you specify the -l option, gsutil will output additional information
  about each matching provider, bucket, subdirectory, or object. For example:

    gsutil ls -l gs://bucket/*.txt

  will print the object size, creation time stamp, and name of each matching
  object, along with the total count and sum of sizes of all matching objects:

       2276224  2017-03-02T19:25:17Z  gs://bucket/obj1
       3914624  2017-03-02T19:30:27Z  gs://bucket/obj2
    TOTAL: 2 objects, 6190848 bytes (5.9 MiB)

  Note that the total listed in parentheses above is in mebibytes (or gibibytes,
  tebibytes, etc.), which corresponds to the unit of billing measurement for
  Google Cloud Storage.

  You can get a listing of all the objects in the top-level bucket directory
  (along with the total count and sum of sizes) using a command like:

    gsutil ls -l gs://bucket

  To print additional detail about objects and buckets use the gsutil ls -L
  option. For example:

    gsutil ls -L gs://bucket/obj1

  will print something like:

    gs://bucket/obj1:
            Creation time:                    Fri, 26 May 2017 22:55:44 GMT
            Update time:                      Tue, 18 Jul 2017 12:31:18 GMT
            Storage class:                    MULTI_REGIONAL
            Content-Length:                   60183
            Content-Type:                     image/jpeg
            Hash (crc32c):                    zlUhtg==
            Hash (md5):                       Bv86IAzFzrD1Z2io/c7yqA==
            ETag:                             5ca67960a586723b7344afffc81
            Generation:                       1378862725952000
            Metageneration:                   1
            ACL:                              [
      {
        "entity": "project-owners-867484910061",
        "projectTeam": {
          "projectNumber": "867484910061",
          "team": "owners"
        },
        "role": "OWNER"
      },
      {
        "email": "jane@gmail.com",
        "entity": "user-jane@gmail.com",
        "role": "OWNER"
      }
    ]
    TOTAL: 1 objects, 60183 bytes (58.77 KiB)

  Note that results may contain additional fields, such as custom metadata or
  a storage class update time, if they are applicable to the object.

  Also note that some fields, such as update time, are not available with the
  (non-default) XML API.

  See also "gsutil help acl" for getting a more readable version of the ACL.


<B>LISTING BUCKET DETAILS</B>
  If you want to see information about the bucket itself, use the -b
  option. For example:

    gsutil ls -L -b gs://bucket

  will print something like:

    gs://bucket/ :
            Storage class:                MULTI_REGIONAL
            Location constraint:          US
            Versioning enabled:           False
            Logging configuration:        None
            Website configuration:        None
            CORS configuration:           None
            Lifecycle configuration:      None
            Requester Pays enabled:       True
            Labels:                       None
            Default KMS key:              None
            Time created:                 Thu, 14 Jan 2016 19:25:17 GMT
            Time updated:                 Thu, 08 Jun 2017 21:17:59 GMT
            Metageneration:               1
            Bucket Policy Only enabled:   False
            ACL:
              [
                {
                  "entity": "project-owners-867489160491",
                  "projectTeam": {
                    "projectNumber": "867489160491",
                    "team": "owners"
                  },
                  "role": "OWNER"
                }
              ]
            Default ACL:
              [
                {
                  "entity": "project-owners-867489160491",
                  "projectTeam": {
                    "projectNumber": "867489160491",
                    "team": "owners"
                  },
                  "role": "OWNER"
                }
              ]

  Note that some fields above (time created, time updated, metageneration) are
  not available with the (non-default) XML API.


<B>OPTIONS</B>
  -l          Prints long listing (owner, length).

  -L          Prints even more detail than -l.  Note: If you use this option
              with the (non-default) XML API it will generate an additional
              request per object being listed, which makes the -L option run
              much more slowly (and cost more) using the XML API than the
              default JSON API.

  -d          List matching subdirectory names instead of contents, and do not
              recurse into matching subdirectories even if the -R option is
              specified.

  -b          Prints info about the bucket when used with a bucket URL.

  -h          When used with -l, prints object sizes in human readable format
              (e.g., 1 KiB, 234 MiB, 2 GiB, etc.)

  -p proj_id  Specifies the project ID to use for listing buckets.

  -R, -r      Requests a recursive listing, performing at least one listing
              operation per subdirectory. If you have a large number of
              subdirectories and do not require recursive-style output ordering,
              you may be able to instead use wildcards to perform a flat
              listing, e.g.  `gsutil ls gs://mybucket/**`, which will generally
              perform fewer listing operations.

  -a          Includes non-current object versions / generations in the listing
              (only useful with a versioning-enabled bucket). If combined with
              -l option also prints metageneration for each listed object.

  -e          Include ETag in long listing (-l) output.
""")


class ListingStyle(object):
  """Enum class for specifying listing style."""
  SHORT = 'SHORT'
  LONG = 'LONG'
  LONG_LONG = 'LONG_LONG'


class LsCommand(Command):
  """Implementation of gsutil ls command."""

  # Command specification. See base class for documentation.
  command_spec = Command.CreateCommandSpec(
      'ls',
      command_name_aliases=['dir', 'list'],
      usage_synopsis=_SYNOPSIS,
      min_args=0,
      max_args=NO_MAX,
      supported_sub_args='aebdlLhp:rR',
      file_url_ok=False,
      provider_url_ok=True,
      urls_start_arg=0,
      gs_api_support=[ApiSelector.XML, ApiSelector.JSON],
      gs_default_api=ApiSelector.JSON,
      argparse_arguments=[
          CommandArgument.MakeZeroOrMoreCloudURLsArgument()
      ]
  )
  # Help specification. See help_provider.py for documentation.
  help_spec = Command.HelpSpec(
      help_name='ls',
      help_name_aliases=['dir', 'list'],
      help_type='command_help',
      help_one_line_summary='List providers, buckets, or objects',
      help_text=_DETAILED_HELP_TEXT,
      subcommand_help_text={},
  )

  def _PrintBucketInfo(self, bucket_blr, listing_style):
    """Print listing info for given bucket.

    Args:
      bucket_blr: BucketListingReference for the bucket being listed
      listing_style: ListingStyle enum describing type of output desired.

    Returns:
      Tuple (total objects, total bytes) in the bucket.
    """
    if (listing_style == ListingStyle.SHORT or
        listing_style == ListingStyle.LONG):
      print bucket_blr
      return
    # listing_style == ListingStyle.LONG_LONG:
    # We're guaranteed by the caller that the root object is populated.
    bucket = bucket_blr.root_object
    location_constraint = bucket.location
    storage_class = bucket.storageClass
    fields = {'bucket': bucket_blr.url_string,
              'storage_class': storage_class,
              'location_constraint': location_constraint,
              'acl': AclTranslation.JsonFromMessage(bucket.acl),
              'default_acl': AclTranslation.JsonFromMessage(
                  bucket.defaultObjectAcl)}

    fields['versioning'] = bucket.versioning and bucket.versioning.enabled
    fields['website_config'] = 'Present' if bucket.website else 'None'
    fields['logging_config'] = 'Present' if bucket.logging else 'None'
    fields['cors_config'] = 'Present' if bucket.cors else 'None'
    fields['lifecycle_config'] = 'Present' if bucket.lifecycle else 'None'
    fields['requester_pays'] = bucket.billing and bucket.billing.requesterPays
    if bucket.retentionPolicy:
      fields['retention_policy'] = 'Present'
    if bucket.labels:
      fields['labels'] = LabelTranslation.JsonFromMessage(
          bucket.labels, pretty_print=True)
    else:
      fields['labels'] = 'None'
    if bucket.encryption and bucket.encryption.defaultKmsKeyName:
      fields['default_kms_key'] = bucket.encryption.defaultKmsKeyName
    else:
      fields['default_kms_key'] = 'None'
    fields['encryption_config'] = 'Present' if bucket.encryption else 'None'
    # Fields not available in all APIs (e.g. the XML API)
    if bucket.metageneration:
      fields['metageneration'] = bucket.metageneration
    if bucket.timeCreated:
      fields['time_created'] = bucket.timeCreated.strftime(
          '%a, %d %b %Y %H:%M:%S GMT')
    if bucket.updated:
      fields['updated'] = bucket.updated.strftime('%a, %d %b %Y %H:%M:%S GMT')
    if bucket.defaultEventBasedHold:
      fields['default_eventbased_hold'] = bucket.defaultEventBasedHold
    if bucket.iamConfiguration and bucket.iamConfiguration.bucketPolicyOnly:
      enabled = bucket.iamConfiguration.bucketPolicyOnly.enabled
      fields['bucket_policy_only_enabled'] = enabled

    # For field values that are multiline, add indenting to make it look
    # prettier.
    for key in fields:
      previous_value = fields[key]
      if (not isinstance(previous_value, basestring) or
          '\n' not in previous_value):
        continue
      new_value = previous_value.replace('\n', '\n\t  ')
      # Start multiline values on a new line if they aren't already.
      if not new_value.startswith('\n'):
        new_value = '\n\t  ' + new_value
      fields[key] = new_value

    # Only display certain properties if the given API returned them (JSON API
    # returns many fields that the XML API does not).
    metageneration_line = ''
    time_created_line = ''
    time_updated_line = ''
    default_eventbased_hold_line = ''
    retention_policy_line = ''
    bucket_policy_only_enabled_line = ''
    if 'metageneration' in fields:
      metageneration_line = '\tMetageneration:\t\t\t{metageneration}\n'
    if 'time_created' in fields:
      time_created_line = '\tTime created:\t\t\t{time_created}\n'
    if 'updated' in fields:
      time_updated_line = '\tTime updated:\t\t\t{updated}\n'
    if 'default_eventbased_hold' in fields:
      default_eventbased_hold_line = (
          '\tDefault Event-Based Hold:\t{default_eventbased_hold}\n')
    if 'retention_policy' in fields:
      retention_policy_line = '\tRetention Policy:\t\t{retention_policy}\n'
    if 'bucket_policy_only_enabled' in fields:
      bucket_policy_only_enabled_line = ('\tBucket Policy Only enabled:\t'
                                         '{bucket_policy_only_enabled}\n')


    print(('{bucket} :\n'
           '\tStorage class:\t\t\t{storage_class}\n'
           '\tLocation constraint:\t\t{location_constraint}\n'
           '\tVersioning enabled:\t\t{versioning}\n'
           '\tLogging configuration:\t\t{logging_config}\n'
           '\tWebsite configuration:\t\t{website_config}\n'
           '\tCORS configuration: \t\t{cors_config}\n'
           '\tLifecycle configuration:\t{lifecycle_config}\n'
           '\tRequester Pays enabled:\t\t{requester_pays}\n' +
           retention_policy_line +
           default_eventbased_hold_line +
           '\tLabels:\t\t\t\t{labels}\n' +
           '\tDefault KMS key:\t\t{default_kms_key}\n' +
           time_created_line +
           time_updated_line +
           metageneration_line +
           bucket_policy_only_enabled_line +
           '\tACL:\t\t\t\t{acl}\n'
           '\tDefault ACL:\t\t\t{default_acl}').format(**fields))
    if bucket_blr.storage_url.scheme == 's3':
      print('Note: this is an S3 bucket so configuration values may be '
            'blank. To retrieve bucket configuration values, use '
            'individual configuration commands such as gsutil acl get '
            '<bucket>.')

  def _PrintLongListing(self, bucket_listing_ref):
    """Prints an object with ListingStyle.LONG."""
    obj = bucket_listing_ref.root_object
    url_str = bucket_listing_ref.url_string
    if (obj.metadata and S3_DELETE_MARKER_GUID in
        obj.metadata.additionalProperties):
      size_string = '0'
      num_bytes = 0
      num_objs = 0
      url_str += '<DeleteMarker>'
    else:
      size_string = (MakeHumanReadable(obj.size)
                     if self.human_readable else str(obj.size))
      num_bytes = obj.size
      num_objs = 1

    timestamp = JSON_TIMESTAMP_RE.sub(
        r'\1T\2Z', str(obj.timeCreated).decode(UTF8).encode('ascii'))
    printstr = '%(size)10s  %(timestamp)s  %(url)s'
    encoded_etag = None
    encoded_metagen = None
    if self.all_versions:
      printstr += '  metageneration=%(metageneration)s'
      encoded_metagen = str(obj.metageneration).encode(UTF8)
    if self.include_etag:
      printstr += '  etag=%(etag)s'
      encoded_etag = obj.etag.encode(UTF8)
    format_args = {
        'size': size_string,
        'timestamp': timestamp,
        'url': url_str.encode(UTF8),
        'metageneration': encoded_metagen,
        'etag': encoded_etag
    }
    print printstr % format_args
    return (num_objs, num_bytes)

  def RunCommand(self):
    """Command entry point for the ls command."""
    got_nomatch_errors = False
    got_bucket_nomatch_errors = False
    listing_style = ListingStyle.SHORT
    get_bucket_info = False
    self.recursion_requested = False
    self.all_versions = False
    self.include_etag = False
    self.human_readable = False
    self.list_subdir_contents = True
    if self.sub_opts:
      for o, a in self.sub_opts:
        if o == '-a':
          self.all_versions = True
        elif o == '-e':
          self.include_etag = True
        elif o == '-b':
          get_bucket_info = True
        elif o == '-h':
          self.human_readable = True
        elif o == '-l':
          listing_style = ListingStyle.LONG
        elif o == '-L':
          listing_style = ListingStyle.LONG_LONG
        elif o == '-p':
          # Project IDs are sent as header values when using gs and s3 XML APIs.
          InsistAscii(a, 'Invalid non-ASCII character found in project ID')
          self.project_id = a
        elif o == '-r' or o == '-R':
          self.recursion_requested = True
        elif o == '-d':
          self.list_subdir_contents = False

    if not self.args:
      # default to listing all gs buckets
      self.args = ['gs://']

    total_objs = 0
    total_bytes = 0

    def MaybePrintBucketHeader(blr):
      if len(self.args) > 1:
        print '%s:' % blr.url_string.encode(UTF8)
    print_bucket_header = MaybePrintBucketHeader

    for url_str in self.args:
      storage_url = StorageUrlFromString(url_str)
      if storage_url.IsFileUrl():
        raise CommandException('Only cloud URLs are supported for %s'
                               % self.command_name)
      bucket_fields = None
      if (listing_style == ListingStyle.SHORT or
          listing_style == ListingStyle.LONG):
        bucket_fields = ['id']
      elif listing_style == ListingStyle.LONG_LONG:
        bucket_fields = ['acl',
                         'billing',
                         'cors',
                         'defaultObjectAcl',
                         'encryption',
                         'iamConfiguration',
                         'labels',
                         'location',
                         'logging',
                         'lifecycle',
                         'metageneration',
                         'retentionPolicy',
                         'defaultEventBasedHold',
                         'storageClass',
                         'timeCreated',
                         'updated',
                         'versioning',
                         'website']
      if storage_url.IsProvider():
        # Provider URL: use bucket wildcard to list buckets.
        for blr in self.WildcardIterator(
            '%s://*' % storage_url.scheme).IterBuckets(
                bucket_fields=bucket_fields):
          self._PrintBucketInfo(blr, listing_style)
      elif storage_url.IsBucket() and get_bucket_info:
        # ls -b bucket listing request: List info about bucket(s).
        total_buckets = 0
        for blr in self.WildcardIterator(url_str).IterBuckets(
            bucket_fields=bucket_fields):
          if not ContainsWildcard(url_str) and not blr.root_object:
            # Iterator does not make an HTTP call for non-wildcarded
            # listings with fields=='id'. Ensure the bucket exists by calling
            # GetBucket.
            self.gsutil_api.GetBucket(
                blr.storage_url.bucket_name,
                fields=['id'], provider=storage_url.scheme)
          self._PrintBucketInfo(blr, listing_style)
          total_buckets += 1
        if not ContainsWildcard(url_str) and not total_buckets:
          got_bucket_nomatch_errors = True
      else:
        # URL names a bucket, object, or object subdir ->
        # list matching object(s) / subdirs.
        def _PrintPrefixLong(blr):
          print '%-33s%s' % ('', blr.url_string.encode(UTF8))

        if listing_style == ListingStyle.SHORT:
          # ls helper by default readies us for a short listing.
          listing_helper = LsHelper(
              self.WildcardIterator, self.logger,
              all_versions=self.all_versions,
              print_bucket_header_func=print_bucket_header,
              should_recurse=self.recursion_requested,
              list_subdir_contents=self.list_subdir_contents)
        elif listing_style == ListingStyle.LONG:
          bucket_listing_fields = ['name', 'timeCreated', 'updated', 'size']
          if self.all_versions:
            bucket_listing_fields.extend(['generation', 'metageneration'])
          if self.include_etag:
            bucket_listing_fields.append('etag')

          listing_helper = LsHelper(
              self.WildcardIterator, self.logger,
              print_object_func=self._PrintLongListing,
              print_dir_func=_PrintPrefixLong,
              print_bucket_header_func=print_bucket_header,
              all_versions=self.all_versions,
              should_recurse=self.recursion_requested,
              fields=bucket_listing_fields,
              list_subdir_contents=self.list_subdir_contents)

        elif listing_style == ListingStyle.LONG_LONG:
          # List all fields
          bucket_listing_fields = (UNENCRYPTED_FULL_LISTING_FIELDS +
                                   ENCRYPTED_FIELDS)
          listing_helper = LsHelper(
              self.WildcardIterator, self.logger,
              print_object_func=PrintFullInfoAboutObject,
              print_dir_func=_PrintPrefixLong,
              print_bucket_header_func=print_bucket_header,
              all_versions=self.all_versions,
              should_recurse=self.recursion_requested,
              fields=bucket_listing_fields,
              list_subdir_contents=self.list_subdir_contents)
        else:
          raise CommandException('Unknown listing style: %s' % listing_style)

        exp_dirs, exp_objs, exp_bytes = (
            listing_helper.ExpandUrlAndPrint(storage_url))
        if storage_url.IsObject() and exp_objs == 0 and exp_dirs == 0:
          got_nomatch_errors = True
        total_bytes += exp_bytes
        total_objs += exp_objs

    if total_objs and listing_style != ListingStyle.SHORT:
      print ('TOTAL: %d objects, %d bytes (%s)' %
             (total_objs, total_bytes, MakeHumanReadable(float(total_bytes))))
    if got_nomatch_errors:
      raise CommandException('One or more URLs matched no objects.')
    if got_bucket_nomatch_errors:
      raise NotFoundException('One or more bucket URLs matched no buckets.')

    return 0
