lsPP

#!/usr/bin/env python3
#
# lss: A clone of bash "ls", but more flexible. I hope.
# 2015-03-19: Written by Steven J. DeRose.
#
import sys
import os
import argparse
import re
import subprocess
import codecs

from alogging import ALogger

args = None
lg = ALogger()

__metadata__ = {
    "title"        : "lss",
    "description"  : "A clone of bash 'ls', but more flexible. I hope.",
    "rightsHolder" : "Steven J. DeRose",
    "creator"      : "http://viaf.org/viaf/50334488",
    "type"         : "http://purl.org/dc/dcmitype/Software",
    "language"     : "Python 3.7",
    "created"      : "2015-03-19",
    "modified"     : "2020-09-22",
    "publisher"    : "http://github.com/sderose",
    "license"      : "https://creativecommons.org/licenses/by-sa/3.0/"
}
__version__ = __metadata__["modified"]


descr = """
=Description=

A replacement for "ls"; hopefully more flexible and less confusing.

Unfinished. For now, it just collects and reports everything in site;
no selection or layout.

=Capabilities=

* Choice of bare filenames, relative paths, or absolute.

* Names for all the data items about files, allowing symmetrical
operations on any/all of them.

* Controllable selection, order, format, and width of items.

* Output in multiple useful data representations


=Notes=

    cf lsoutline!

    mnemonic types vs. flags
        (including "file" output)

    Fields headers

    Separate control of fields to show:
        %name:width

    Option to shorten to ~


=Mnemonic names of several kinds=

==Named Fields (aka features):==

These are the properties known about any given file. They all have very short
mnemonic names, which are used to refer to them in options. For example, you
can specify whether (and where and how wide) each field is displayed; whether
to include or exclude files with certain properties; and so on.

===Name===

* B<NX>

The basename of the file, including the extension.

* B<NAME>

The basename of the file, B<not> including the extension.

* B<EXT>

The extension of the file

* B<SUF>

Any suffix, defined as any non-alphanumeric characters following the extension
(if any). Most commonly, this is a backup-file indicator such as "~" or "#"

* B<BAK> (1 or 0)

Whether the file appears to be a backup file. This is
determined by checking for a variety of naming conventions.

    X~ #X# X.bak
    Backup (n) of X
    Backup of backup of X
    Copy (n) of X
    Copy of copyu of X
    X backup

* B<BNUM> (int)

If the file appears to be a backup, a best guess at its generation number.
For example, both "Backup 2 of X" and "Backup of backup of X" yield 2.

===Dates===

* B<CRE>

The creation date of the file. See L<Dates>, below, for more information.

* B<MOD>

The modification date of the file. See L<Dates>, below, for more information.

* B<ACC>

The last-accessed date of the file. See L<Dates>, below, for more information.

===Permissions===

* B<UR UW UX GR GW GX OR OW OX> (1 or 0)

Individual permission flags for each of user/group/other crossed
with read, write, and execute.

* B<PERM> (octal int)

A numeric permission value, *nix-style.

* B<USER>

The user name of the owner of the file.

* B<UNUM>

The user number of the owner of the file.

* B<GROUP>

The group name applicable to the file.

* B<GNUM>

The group name applicable to the file.

===Type===

(This should also get additions for the data from C<file>)

=over

* B<TEXT> (1 or 0)

Whether the file appears to be readable text.

* B<BIN> (1 or 0)

Whether the file appears to be "binary" (not TEXT).

* B<LINK> (HARD, SOFT, ALIAS, or NONE)

Whether the items is a link, and if so, what kind.
Alias represents an OS-specific type that is not a hard or soft *nix link.
It would be better to identify and distinguish at least Mac and Win forms.

===Other===

* B<SIZE> (int)

* B<VER> (SVN, GIT, CVS, or NONE)

Whether the file appears to be under a version-control system.
Typically, this is indicated by a hidden file or directory in the same
or an ancestor directory. Other values may be added.

* B<REC> (NIX, MAC, or DOS)

The record-separator type found in the file (if the file is TEXT).

* B<HIDDEN> (1 or 0)

True if the file is GUI-invisible or its name begins with ".".

* B<TGT>

The "real" file that this item points to, if this item is a link.

* B<INODE> (int)

* B<DEV>

The device on which the file resides.

* B<DFMT> (string)

The format of the device on which the file resides.
Typical values could be FAT, FAT32, EXT3, EXT4.

* B<META>

A named meta-data field for the file.
This includes things like the source URI for a downloaded file;
many properties of image files; etc.

* MORE:

    NLINKS
    INODE protection mode
    setuid
    setgid
    sticky bit
    permission mask

    cf https://docs.python.org/2/library/stat.html
    nodump
    immutable
    append
    opaque
    nounlink
    compressed (and cf zip/bzip/etc.)
    archived
    snapshot
    ?disk image

    What is "ls --author"?


==Date and time formats==

See also C<man date>, which does formats via C<strftime>, which is a superset
of L<ISO/IEC 9899:1990>, and available in Python (see
L<https://docs.python.org/2/library/time.html#time.strftime>. May as well stick
with the same thing.


    1990-12-31 @ 23:15:02 CDT
    raw seconds

     %A    localized full weekday name.
     %a    localized abbreviated weekday name.
     %B    localized full month name.
     %b    localized abbreviated month name.
     %C    (year / 100); single digits are preceded by a blank
     %c    localized time and date.
     %D    is equivalent to ``%m/%d/%y''.
     %d    the day of the month as a decimal number (01-31).
     %E* %O* POSIX locale extensions.
     %e    day of the month (1-31); single digits are preceded by a blank.
     %F    is equivalent to ``%Y-%m-%d''.
     %G    a year as a decimal number with century.
     %g    the same year as in ``%G'', but without century (00-99).
     %H    the hour (24-hour clock) as a decimal number (00-23).
     %h    the same as %b.
     %I    the hour (12-hour clock) as a decimal number (01-12).
     %j    the day of the year as a decimal number (001-366).
     %k    the hour (24-hour clock) as a decimal number (' 0'-'23')
     %l    the hour (12-hour clock) as a decimal number (' 1'-'12')
     %M    the minute as a decimal number (00-59).
     %m    the month as a decimal number (01-12).
     %n    a newline.
     %O*   the same as %E*.
     %p    localized (a.m.) or (p.m.)  as appropriate.
     %R    is equivalent to ``%H:%M''.
     %r    is equivalent to ``%I:%M:%S %p''.
     %S    the second as a decimal number (00-60).
     %s    the number of seconds since the Epoch, UTC (see mktime(3)).
     %T    is equivalent to ``%H:%M:%S''.
     %t    a tab.
     %U    the week number of the year (start Sunday), 00-53.
     %u    the weekday (Monday = 1)
     %V    the week number of the year (start Monday), 01-53.
     %v    is equivalent to ``%e-%b-%Y''.
     %W    the week number of the year (start Monday), 00-53.
     %w    the weekday (start Sunday) (0-6).
     %X    localizedtime.
     %x    localized date.
     %Y    the year with century.
     %y    the year without century, 00-99.
     %Z    the time zone name.
     %z    the time zone offset from UTC; +east, -west of UTC, hhmm (RFC 822)
     %+    localized date and time (similar to date(1)).
     %-*   GNU libc extension.  No padding when performing numerical outputs.
     %_*   GNU libc extension.  Explicitly specify space for padding.
     %0*   GNU libc extension.  Explicitly specify zero for padding.
     %%    `%'.


==File "types"==

socket
symlink
regular file
block device
directory
character device
fifo
whiteout


==META names==


=Known bugs and Limitations=

Does not yet distinguish real vs. effective uid and gid.

META needs a way to refer to both name and value, and a list of useful ones.


=History=

  2015-03-19: Written. Copyright by Steven J. DeRose.
  2016-11-03, 2019-10-29: Lint.
  2020-09-22: New layout, more lint.


=To do=


=Rights=

Copyright 2015-03-19 by Steven J. DeRose. This work is licensed under a
Creative Commons Attribution-Share-alike 3.0 unported license.
For further information on this license, see
[https://creativecommons.org/licenses/by-sa/3.0].

For the most recent version, see [http://www.derose.net/steve/utilities]
or [https://github.com/sderose].


=Options=
"""


fieldInfo = {
}
fieldNames = fieldInfo.keys()

statNames = [
    "ST_MODE",
    "ST_INO",
    "ST_DEV",
    "ST_NLINK",
    "ST_UID",
    "ST_GID",
    "ST_SIZE",
    "ST_ATIME",
    "ST_MTIME",
    "ST_CTIME",
]

# These are derived from the "dircolors" command, and bash LS_COLORS
fileCategories = [
    "archive",
    "audio",
    "block_device",
    "capability",
    "character_device",
    "directory",
    "executable",
    "orphan",
    "picture",
    "pipe",
    "setGID",
    "setUID",
    "socket",
    "sticky_directory",
    "sticky_writable",
    "symlink",
    "video",
]


# Uses the 'file' command, which may be controlled by /etc/magic.
# Category names can have various
# modifiers.... There's no evident grammar.
#
fileCommandCategories = {
    # MAIN TYPE     : SUBTYPES (prefixed)
    "byte-compiled" : [ "python 2.7", ],
    "data"          : [ "", "compressed", "image", "timezone",
                        "Java Jar file", ],
    "diff output"   : [ ],
    "directory"     : [ ],
    "empty"         : [ ],
    "executable"    : [ ],
    "library file"  : [ "libtool", ],
    "filesystem"    : [ ],
    "program"       : [ "^codeLanguage", ],
    "regular file"  : [ ],
    "script"        : [ "", "shell", ], # plus ^scriptLanguage before that
    "setgid"        : [ ],              # no comma after
    "setuid"        : [ ],              # no comma after
    "socket"        : [ ],
    "source"        : [ "^sourceLanguage", ],
    "special"       : [ "block", "character" ],
    "sticky"        : [ ],              # no comma after
    "symbolic link" : [ "", "broken", ],
    "text"          : [ "", "document" ],
}

fileCommandPrefixes = {
    "charset": {
        "ASCII",
        "UTF-8 Unicode",
        "ISO-8859",
        "Non-ISO extended-ASCII",
    },

    "codeLanguage": {
        "C",               # or "program text"
        "C++",
        "Python",
        "Pascal",
    },

    "sourceLanguage": {
        "LISP/Scheme",
        "Perl5 module",
    },

    "scriptLanguage": {
        "awk",
        "Bourne-Again",
        "POSIX",
        "PHP",
        "Perl",
        "M4 macro processor",
    },

    "documentLanguage": {
        "HTML",
        "xHTML",
        "XML",
        "exported XML",
        "exported SGML",
    },

    "compression": {
        "Zip",
        "gzip",
        "XZ",
        # tar, Jar, hqx
    },

    "imageLanguage": {
        "Netpbm PBM",
        "Netpbm PGM",
        "Netpbm PPM",
        "PNG",
        "X pixmap",
        "JPEG", ###
    },

}

fileCommandSuffixes = [
    "with escape sequences",
    "with very long lines",

    "with CRLF",
    "with LF",
    "with CR",

    "with CR line terminators",
    "with LF line terminators",
    "with CRLF line terminators",
    "with no line terminators",
    "with NEL line terminators",
    "no read permission",
]

dataFromFile = """
    text        -> charset, lineends, lineLengthWarning
        code    -> language, version
            script
        markup  -> format, version, schema
    image      -> format, depth, size, colorspace, interlace
    executable -> platform, format, version, linkType, source/object
    appdata    -> appName
    appoutput  -> appName
    system     -> (pipe/special/...)
    compressed -> format
"""


# Examples (BSD and MacOSX)
"""
a /usr/bin/perl script text executable
Bourne-Again shell script text executable
Mach-O 64-bit executable x86_64

Lisp/Scheme program text

RIFF (little-endian) data, WAVE audio, Microsoft PCM, 8 bit, mono 11025 Hz
MPEG ADTS, layer III,  v2.5, 128 kbps, 11.025 kHz, Monaural

version...

, 11024 Hz)
, 155 x 127
, 8-bit/color RGB
, 8-bit/color RGBA
, ANSI
, AVI
, EXIF standard 2.21
, JFIF standard 1.01
, Little Endian
, MPEG v4 system
, at least v2.0 to extract
, block size = 900k
, from Unix
, max compression
, non-interlaced
, v1d
, ~15 fps

CDF V2 Document
ISO Media
Macromedia Flash Video
Microsoft ASF
Mobipocket E-book "The_question-f_King_Milinda"
OpenDocument Drawing
OpenDocument Presentation
OpenDocument Spreadsheet
OpenDocument Text

JPEG image data
NeXT/Apple typedstream data
Ogg data
PNG image data
RAR archive data
RIFF (little-endian) data
Rich Text Format data
TrueType font data
bzip2 compressed data
data

PDF document

VAX COFF executable not stripped - version 16959
a /usr/bin/perl -w script text executable

RealMedia file
Audio file with ID3 version 2.2.0
BitTorrent file

UTF-8 Unicode English text
ASCII C++ program text
ASCII English text
ASCII news text
HTML document text
ISO-8859 English text
exported SGML document text
XML  document text
xHTML document text
xar archive - version 1
PostScript document text conforming DSC level 3.0

"""

# These follow ", ", and are followed by ": " and some value.
#
macFields = {
    "Author",
    "Code page",
    "Create Time/Date",
    "Keywords",
    "Last Printed",
    "Last Saved By",
    "Last Saved Time/Date",
    "Name of Creating Application",
    "Number of Characters",
    "Number of Pages",
    "Number of Words",
    "Os",
    "Revision Number",
    "Security",
    "Template",
    "Title",
    "Total Editing Time",
    "audio",
    "comment",
    "contains",
    "last modified",
    "video",
}


###############################################################################
#
def processOptions():
    #x = sys.argv[0]
    try:
        from BlockFormatter import BlockFormatter
        parser = argparse.ArgumentParser(
            description=descr, formatter_class=BlockFormatter)
    except ImportError:
        parser = argparse.ArgumentParser(description=descr)

    parser.add_argument(
        "--extension",        type=str, metavar="E", default="",
        help='Only process input files with this extension. Default: "" (=all).')
    parser.add_argument(
        "--recursive",        action="store_true",
        help="Traverse subdirectories.")
    parser.add_argument(
        "--exclude",          type=str, action="append", metavar="R",
        help="Do not include files matching regex. Repeatable.")
    parser.add_argument(
        "--include",          type=str, action="append", metavar="R",
        help="Include files matching regex (overrides --exclude). Repeatable.")

    # DISPLAY
    #
    parser.add_argument(
        "--color",  # Don't default. See below.
        help="Colorize the output.")
    parser.add_argument(
        "--colorMap",         type=str, metavar="F",
        help="Use this file to specify how to colorize.")
    parser.add_argument(
        "--ignoreCase", "--ignore-case", "-i", action="store_true",
        help="Disregard case distinctions when sorting.")
    parser.add_argument(
        "--interleave",       action="store_true",
        help="Sort directories and files together, not separately")
    parser.add_argument(
        "--oformat", "--output-format",
        type=str, metavar="F", choices=[ "TSV", "CSV", "JSON", "XSV", "ARRAY" ],
        help="Use this format for output data.")
    parser.add_argument(
        "--paths",            type=str, metavar="P",
        choices=[ "ABS", "REL", "NONE", ],
        help="Use this character set for output files.")
    parser.add_argument(
        "--size",             type=str,
        choices = [ "DEC", "HEX", "k", "K", "COM" ],
        help="How to represent the file-size.")
    parser.add_argument(
        "--sortby", "--orderby", type=str, action="append", metavar="F",
        choices = fieldNames,
        help="Sort results by this field. Repeatable.")

    # MISCELLANEOUS
    #
    parser.add_argument(
        "--quiet", "-q",      action="store_true",
        help="Suppress most messages.")
    parser.add_argument(
        "--verbose", "-v",    action="count",       default=0,
        help="Add more messages (repeatable).")
    parser.add_argument(
        "--version",          action="version",     version="Version of "+__version__,
        help="Display version information, then exit.")

    parser.add_argument(
        "files",             type=str,
        nargs=argparse.REMAINDER,
        help="Path(s) to input file(s)")

    args0 = parser.parse_args()
    lg.setVerbose(args0.verbose)
    if (args0.color is None):
        args0.color = ("CLI_COLOR" in os.environ and sys.stderr.isatty())
    lg.setColors(args0.color)
    #sys.argv = [x] + args0.files
    return(args0)


###############################################################################
#
def doOneItem(path):
    """Recurse if needed for dirs; pass files off for handling.
    """
    if (isExcluded(path)): return
    lg.info1("====Starting item '%s'" % (path))
    recnum = 0
    if (not os.path.exists(path)):
        lg.error("Couldn't find '%s'." % (path))
    elif (os.path.isdir(path)):
        lg.bumpStat("totalDirs")
        if (args.recursive):
            for child in os.listdir(path):
                recnum += doOneItem(os.path.join(path,child))
    else:
        doOneFile(path)
    return(recnum)


###############################################################################
#
def doOneFile(path):
    """See if a specific file (not directory) is to be included,
    and if so, report all the gory details.
    """
    lg.bumpStat("totalFiles")

    ########################################
    # os.stat information
    st = os.stat(path)
    for key in (statNames):
        displayName, value = getStatItem(st, key)
        lg.pline(displayName, value)

    ########################################
    # "file" command information
    fileInfo =  subprocess.check_output([ "file", path ])
    fileParts = re.split(r",\s*", fileInfo)
    #fileLinkTarget = ""
    for fp in (fileParts):
        mat = re.match(r"symbolic link to (.*)", fp)
        if (not mat): continue
        #fileLinkTarget = mat.group(1)

    ########################################
    # Mac file-system metadata, KMD...

    ########################################
    # Do we need to read the actual content?
    # Maybe to get countTags, countChars, vocab, etc.
    recnum = 0
    if (args.openEverything):
        try:
            fh = codecs.open(path, mode="r", encoding=args.iencoding)
        except IOError as e:
            lg.error("IOError: %s" % (e))
            return(0)
        #recnum = fh.read()
        fh.close()
    return recnum

def isExcluded(path):
    """Filter out files we're not support to deal with
    """
    _, ext = os.path.splitext(path)
    if (args.extension != "" and ext != args.extension):
        lg.bumpStat("wrongExtension")
        return False
    if (args.exclude and re.match(args.exclude, path)):
        if (args.include and re.match(args.include, path)):
            lg.bumpStat("fileExcludeOverridden")
        else:
            lg.bumpStat("fileExcluded")
            return False
    # Add options to sort special files types...

###############################################################################
#
def getStatItem(st, item):
    """Given an os.stat object, hand back a field by name.
    """
    if (item == "ST_MODE"):
        return("Permissions mode"   , st.st_mode)
    if (item == "ST_INO"):
        return("inode number"       , st.st_ino)
    if (item == "ST_DEV"):
        return("device"             , st.st_dev)
    if (item == "ST_NLINK"):
        return("# of hard links"    , st.st_nlink)
    if (item == "ST_UID"):
        return("owning user id"     , st.st_uid)
    if (item == "ST_GID"):
        return("owning group id"    , st.st_gid)
    if (item == "ST_SIZE"):
        return("size"               , st.st_size)
    if (item == "ST_ATIME"):
        return("Time last accessed" , st.st_atime)
    if (item == "ST_MTIME"):
        return("Time last modified" , st.st_mtime)
    if (item == "ST_CTIME"):
        return("Time created"       , st.st_ctime)
    return(None)


###############################################################################
# Main
#
print("\nUnfinished\n")

args = processOptions()

if (len(args.files) == 0):
    args.files.append(os.environ["PWD"])

for f in (args.files):
    doOneItem(f)

if (not args.quiet): lg.info("Done.")