#!/usr/bin/env perl

# cloc -- Count Lines of Code {{{1

# Copyright (C) 2006-2017 Al Danial <al.danial@gmail.com>

# First release August 2006

#

# Includes code from:

# - SLOCCount v2.26

# http://www.dwheeler.com/sloccount/

# by David Wheeler.

# - Regexp::Common v2013031301

# http://search.cpan.org/~abigail/Regexp-Common-2013031301/lib/Regexp/Common.pm

# by Damian Conway and Abigail.

# - Win32::Autoglob

# http://search.cpan.org/~sburke/Win32-Autoglob-1.01/Autoglob.pm

# by Sean M. Burke.

# - Algorithm::Diff

# http://search.cpan.org/~tyemq/Algorithm-Diff-1.1902/lib/Algorithm/Diff.pm

# by Tye McQueen.

#

# This program is free software; you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation; either version 2 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

# GNU General Public License for more details:

# <http://www.gnu.org/licenses/gpl.txt>.

#

# 1}}}

my $VERSION = "1.75"; # odd number == beta; even number == stable

my $URL = "github.com/AlDanial/cloc"; # 'https://' pushes header too wide

require 5.006;

# use modules {{{1

use warnings;

use strict;

use Getopt::Long;

use File::Basename;

use File::Temp qw { tempfile tempdir };

use File::Find;

use File::Path;

use File::Spec;

use IO::File;

use List::Util qw( min max );

use Cwd;

use POSIX qw { strftime ceil};

# Parallel::ForkManager isn't in the standard distribution.

# Use it only if installed, and only if --processes=N is given.

# The module load happens in get_max_processes().

my $HAVE_Parallel_ForkManager = 0;

# Digest::MD5 isn't in the standard distribution. Use it only if installed.

my $HAVE_Digest_MD5 = 0;

eval "use Digest::MD5;";

if (defined $Digest::MD5::VERSION) {

$HAVE_Digest_MD5 = 1;

} else {

warn "Digest::MD5 not installed; will skip file uniqueness checks.\n";

}

# Time::HiRes became standard with Perl 5.8

my $HAVE_Time_HiRes = 0;

eval "use Time::HiRes;";

$HAVE_Time_HiRes = 1 if defined $Time::HiRes::VERSION;

my $HAVE_Rexexp_Common;

# Regexp::Common isn't in the standard distribution. It will

# be installed in a temp directory if necessary.

BEGIN {

if (eval "use Regexp::Common;") {

$HAVE_Rexexp_Common = 1;

} else {

$HAVE_Rexexp_Common = 0;

}

}

my $HAVE_Algorith_Diff = 0;

# Algorithm::Diff isn't in the standard distribution. It will

# be installed in a temp directory if necessary.

eval "use Algorithm::Diff qw ( sdiff ) ";

if (defined $Algorithm::Diff::VERSION) {

$HAVE_Algorith_Diff = 1;

} else {

Install_Algorithm_Diff();

}

# print "2 HAVE_Algorith_Diff = $HAVE_Algorith_Diff\n";

# test_alg_diff($ARGV[$#ARGV - 1], $ARGV[$#ARGV]); die;

# die "Hre=$HAVE_Rexexp_Common Had=$HAVE_Algorith_Diff";

# Uncomment next two lines when building Windows executable with perl2exe

# or if running on a system that already has Regexp::Common.

#use Regexp::Common;

#$HAVE_Rexexp_Common = 1;

#perl2exe_include "Regexp/Common/whitespace.pm"

#perl2exe_include "Regexp/Common/URI.pm"

#perl2exe_include "Regexp/Common/URI/fax.pm"

#perl2exe_include "Regexp/Common/URI/file.pm"

#perl2exe_include "Regexp/Common/URI/ftp.pm"

#perl2exe_include "Regexp/Common/URI/gopher.pm"

#perl2exe_include "Regexp/Common/URI/http.pm"

#perl2exe_include "Regexp/Common/URI/pop.pm"

#perl2exe_include "Regexp/Common/URI/prospero.pm"

#perl2exe_include "Regexp/Common/URI/news.pm"

#perl2exe_include "Regexp/Common/URI/tel.pm"

#perl2exe_include "Regexp/Common/URI/telnet.pm"

#perl2exe_include "Regexp/Common/URI/tv.pm"

#perl2exe_include "Regexp/Common/URI/wais.pm"

#perl2exe_include "Regexp/Common/CC.pm"

#perl2exe_include "Regexp/Common/SEN.pm"

#perl2exe_include "Regexp/Common/number.pm"

#perl2exe_include "Regexp/Common/delimited.pm"

#perl2exe_include "Regexp/Common/profanity.pm"

#perl2exe_include "Regexp/Common/net.pm"

#perl2exe_include "Regexp/Common/zip.pm"

#perl2exe_include "Regexp/Common/comment.pm"

#perl2exe_include "Regexp/Common/balanced.pm"

#perl2exe_include "Regexp/Common/lingua.pm"

#perl2exe_include "Regexp/Common/list.pm"

#perl2exe_include "File/Glob.pm"

use Text::Tabs qw { expand };

use Cwd qw { cwd };

use File::Glob;

# 1}}}

# Usage information, options processing. {{{1

my $ON_WINDOWS = 0;

$ON_WINDOWS = 1 if ($^O =~ /^MSWin/) or ($^O eq "Windows_NT");

if ($ON_WINDOWS and $ENV{'SHELL'}) {

if ($ENV{'SHELL'} =~ m{^/}) {

$ON_WINDOWS = 0; # make Cygwin look like Unix

} else {

$ON_WINDOWS = 1; # MKS defines $SHELL but still acts like Windows

}

}

my $NN = chr(27) . "[0m"; # normal

$NN = "" if $ON_WINDOWS or !(-t STDERR); # -t STDERR: is it a terminal?

my $BB = chr(27) . "[1m"; # bold

$BB = "" if $ON_WINDOWS or !(-t STDERR);

my $script = basename $0;

my $brief_usage = "

cloc -- Count Lines of Code

Usage:

$script [options] <file(s)/dir(s)/git hash(es)>

Count physical lines of source code and comments in the given files

(may be archives such as compressed tarballs or zip files) and/or

recursively below the given directories or git commit hashes.

Example: cloc src/ include/ main.c

$script [options] --diff <set1> <set2>

Compute differences of physical lines of source code and comments

between any pairwise combination of directory names, archive

files or git commit hashes.

Example: cloc --diff Python-3.5.tar.xz python-3.6/

$script --help shows full documentation on the options.

http://$URL has numerous examples and more information.

";

my $usage = "

Usage: $script [options] <file(s)/dir(s)/git hash(es)> | <set 1> <set 2> | <report files>

Count, or compute differences of, physical lines of source code in the

given files (may be archives such as compressed tarballs or zip files)

and/or recursively below the given directories.

${BB}Input Options${NN}

--extract-with=<cmd> This option is only needed if cloc is unable

to figure out how to extract the contents of

the input file(s) by itself.

Use <cmd> to extract binary archive files (e.g.:

.tar.gz, .zip, .Z). Use the literal '>FILE<' as

a stand-in for the actual file(s) to be

extracted. For example, to count lines of code

in the input files

gcc-4.2.tar.gz perl-5.8.8.tar.gz

on Unix use

--extract-with='gzip -dc >FILE< | tar xf -'

or, if you have GNU tar,

--extract-with='tar zxf >FILE<'

and on Windows use, for example:

--extract-with=\"\\\"c:\\Program Files\\WinZip\\WinZip32.exe\\\" -e -o >FILE< .\"

(if WinZip is installed there).

--list-file=<file> Take the list of file and/or directory names to

process from <file>, which has one file/directory

name per line. Only exact matches are counted;

relative path names will be resolved starting from

the directory where cloc is invoked.

See also --exclude-list-file.

--vcs=<VCS> Invoke a system call to <VCS> to obtain a list of

files to work on. If <VCS> is 'git', then will

invoke 'git ls-files' to get a file list and

'git submodule status' to get a list of submodules

whose contents will be ignored. See also --git

which accepts git commit hashes and branch names.

If <VCS> is 'svn' then will invoke 'svn list -R'.

The primary benefit is that cloc will then skip

files explicitly excluded by the versioning tool

in question, ie, those in .gitignore or have the

svn:ignore property.

Alternatively <VCS> may be any system command

that generates a list of files.

Note: cloc must be in a directory which can read

the files as they are returned by <VCS>. cloc will

not download files from remote repositories.

'svn list -R' may refer to a remote repository

to obtain file names (and therefore may require

authentication to the remote repository), but

the files themselves must be local.

--unicode Check binary files to see if they contain Unicode

expanded ASCII text. This causes performance to

drop noticeably.

${BB}Processing Options${NN}

--autoconf Count .in files (as processed by GNU autoconf) of

recognized languages. See also --no-autogen.

--by-file Report results for every source file encountered.

--by-file-by-lang Report results for every source file encountered

in addition to reporting by language.

--count-and-diff <set1> <set2>

First perform direct code counts of source file(s)

of <set1> and <set2> separately, then perform a diff

of these. Inputs may be pairs of files, directories,

or archives. If --out or --report-file is given,

three output files will be created, one for each

of the two counts and one for the diff. See also

--diff, --diff-alignment, --diff-timeout,

--ignore-case, --ignore-whitespace.

--diff <set1> <set2> Compute differences in code and comments between

source file(s) of <set1> and <set2>. The inputs

may be any mix of files, directories, archives,

or git commit hashes. Use --diff-alignment to

generate a list showing which file pairs where

compared. See also --count-and-diff, --diff-alignment,

--diff-timeout, --ignore-case, --ignore-whitespace.

--diff-timeout <N> Ignore files which take more than <N> seconds

to process. Default is 10 seconds.

(Large files with many repeated lines can cause

Algorithm::Diff::sdiff() to take hours.)

--follow-links [Unix only] Follow symbolic links to directories

(sym links to files are always followed).

--force-lang=<lang>[,<ext>]

Process all files that have a <ext> extension

with the counter for language <lang>. For

example, to count all .f files with the

Fortran 90 counter (which expects files to

end with .f90) instead of the default Fortran 77

counter, use

--force-lang=\"Fortran 90\",f

If <ext> is omitted, every file will be counted

with the <lang> counter. This option can be

specified multiple times (but that is only

useful when <ext> is given each time).

See also --script-lang, --lang-no-ext.

--force-lang-def=<file> Load language processing filters from <file>,

then use these filters instead of the built-in

filters. Note: languages which map to the same

file extension (for example:

MATLAB/Mathematica/Objective C/MUMPS/Mercury;

Pascal/PHP; Lisp/OpenCL; Lisp/Julia; Perl/Prolog)

will be ignored as these require additional

processing that is not expressed in language

definition files. Use --read-lang-def to define

new language filters without replacing built-in

filters (see also --write-lang-def).

--git Forces the inputs to be interpreted as git targets

(commit hashes, branch names, et cetera) if these

are not first identified as file or directory

names. This option overrides the --vcs=git logic

if this is given; in other words, --git gets its

list of files to work on directly from git using

the hash or branch name rather than from

'git ls-files'. This option can be used with

--diff to perform line count diffs between git

commits, or between a git commit and a file,

directory, or archive. Use -v/--verbose to see

the git system commands cloc issues.

--ignore-whitespace Ignore horizontal white space when comparing files

with --diff. See also --ignore-case.

--ignore-case Ignore changes in case; consider upper- and lower-

case letters equivalent when comparing files with

--diff. See also --ignore-whitespace.

--lang-no-ext=<lang> Count files without extensions using the <lang>

counter. This option overrides internal logic

for files without extensions (where such files

are checked against known scripting languages

by examining the first line for #!). See also

--force-lang, --script-lang.

--max-file-size=<MB> Skip files larger than <MB> megabytes when

traversing directories. By default, <MB>=100.

cloc's memory requirement is roughly twenty times

larger than the largest file so running with

files larger than 100 MB on a computer with less

than 2 GB of memory will cause problems.

Note: this check does not apply to files

explicitly passed as command line arguments.

--no-autogen[=list] Ignore files generated by code-production systems

such as GNU autoconf. To see a list of these files

(then exit), run with --no-autogen list

See also --autoconf.

--original-dir [Only effective in combination with

--strip-comments] Write the stripped files

to the same directory as the original files.

--read-binary-files Process binary files in addition to text files.

This is usually a bad idea and should only be

attempted with text files that have embedded

binary data.

--read-lang-def=<file> Load new language processing filters from <file>

and merge them with those already known to cloc.

If <file> defines a language cloc already knows

about, cloc's definition will take precedence.

Use --force-lang-def to over-ride cloc's

definitions (see also --write-lang-def ).

--script-lang=<lang>,<s> Process all files that invoke <s> as a #!

scripting language with the counter for language

<lang>. For example, files that begin with

#!/usr/local/bin/perl5.8.8

will be counted with the Perl counter by using

--script-lang=Perl,perl5.8.8

The language name is case insensitive but the

name of the script language executable, <s>,

must have the right case. This option can be

specified multiple times. See also --force-lang,

--lang-no-ext.

--sdir=<dir> Use <dir> as the scratch directory instead of

letting File::Temp chose the location. Files

written to this location are not removed at

the end of the run (as they are with File::Temp).

--skip-uniqueness Skip the file uniqueness check. This will give

a performance boost at the expense of counting

files with identical contents multiple times

(if such duplicates exist).

--stdin-name=<file> Give a file name to use to determine the language

for standard input. (Use - as the input name to

receive source code via STDIN.)

--strip-comments=<ext> For each file processed, write to the current

directory a version of the file which has blank

and commented lines removed (in-line comments

persist). The name of each stripped file is the

original file name with .<ext> appended to it.

It is written to the current directory unless

--original-dir is on.

--sum-reports Input arguments are report files previously

created with the --report-file option. Makes

a cumulative set of results containing the

sum of data from the individual report files.

--processes=NUM Sets the maximum number of processes that cloc

uses. If this parameter is set to 0, multi-

processing will be disabled. On Linux and MacOS

systems, cloc creates up to one process per core

by default if a recent version of the

Parallel::ForkManager module is available.

Multiprocessing is disabled by default if cloc

is unable to determine the number of CPU cores.

Multiprocessing is not supported on Windows systems

and on systems which don't have a recent version

of the Parallel::ForkManager module.

--unix Override the operating system autodetection

logic and run in UNIX mode. See also

--windows, --show-os.

--use-sloccount If SLOCCount is installed, use its compiled

executables c_count, java_count, pascal_count,

php_count, and xml_count instead of cloc's

counters. SLOCCount's compiled counters are

substantially faster than cloc's and may give

a performance improvement when counting projects

with large files. However, these cloc-specific

features will not be available: --diff,

--count-and-diff, --strip-comments, --unicode.

--windows Override the operating system autodetection

logic and run in Microsoft Windows mode.

See also --unix, --show-os.

${BB}Filter Options${NN}

--exclude-dir=<D1>[,D2,] Exclude the given comma separated directories

D1, D2, D3, et cetera, from being scanned. For

example --exclude-dir=.cache,test will skip

all files and subdirectories that have /.cache/

or /test/ as their parent directory.

Directories named .bzr, .cvs, .hg, .git, .svn,

and .snapshot are always excluded.

This option only works with individual directory

names so including file path separators is not

allowed. Use --fullpath and --not-match-d=<regex>

to supply a regex matching multiple subdirectories.

--exclude-ext=<ext1>[,<ext2>[...]]

Do not count files having the given file name

extensions.

--exclude-lang=<L1>[,L2,] Exclude the given comma separated languages

L1, L2, L3, et cetera, from being counted.

--exclude-list-file=<file> Ignore files and/or directories whose names

appear in <file>. <file> should have one file

name per line. Only exact matches are ignored;

relative path names will be resolved starting from

the directory where cloc is invoked.

See also --list-file.

--fullpath Modifies the behavior of --match-f, --not-match-f,

and --not-match-d to include the file's path

in the regex, not just the file's basename.

(This does not expand each file to include its

absolute path, instead it uses as much of

the path as is passed in to cloc.)

Note: --match-d always looks at the full

path and therefore is unaffected by --fullpath.

--include-lang=<L1>[,L2,] Count only the given comma separated languages

L1, L2, L3, et cetera.

--match-d=<regex> Only count files in directories matching the Perl

regex. For example

--match-d='/(src|include)/'

only counts files in directories containing

/src/ or /include/. Unlike --not-match-d,

--match-f, and --not-match-f, --match-d always

compares the fully qualified path against the

regex.

--not-match-d=<regex> Count all files except those in directories

matching the Perl regex. Only the trailing

directory name is compared, for example, when

counting in /usr/local/lib, only 'lib' is

compared to the regex.

Add --fullpath to compare parent directories to

the regex.

Do not include file path separators at the

beginning or end of the regex.

--match-f=<regex> Only count files whose basenames match the Perl

regex. For example

--match-f='^[Ww]idget'

only counts files that start with Widget or widget.

Add --fullpath to include parent directories

in the regex instead of just the basename.

--not-match-f=<regex> Count all files except those whose basenames

match the Perl regex. Add --fullpath to include

parent directories in the regex instead of just

the basename.

--skip-archive=<regex> Ignore files that end with the given Perl regular

expression. For example, if given

--skip-archive='(zip|tar(\.(gz|Z|bz2|xz|7z))?)'

the code will skip files that end with .zip,

.tar, .tar.gz, .tar.Z, .tar.bz2, .tar.xz, and

.tar.7z.

--skip-win-hidden On Windows, ignore hidden files.

${BB}Debug Options${NN}

--categorized=<file> Save names of categorized files to <file>.

--counted=<file> Save names of processed source files to <file>.

--diff-alignment=<file> Write to <file> a list of files and file pairs

showing which files were added, removed, and/or

compared during a run with --diff. This switch

forces the --diff mode on.

--explain=<lang> Print the filters used to remove comments for

language <lang> and exit. In some cases the

filters refer to Perl subroutines rather than

regular expressions. An examination of the

source code may be needed for further explanation.

--help Print this usage information and exit.

--found=<file> Save names of every file found to <file>.

--ignored=<file> Save names of ignored files and the reason they

were ignored to <file>.

--print-filter-stages Print processed source code before and after

each filter is applied.

--show-ext[=<ext>] Print information about all known (or just the

given) file extensions and exit.

--show-lang[=<lang>] Print information about all known (or just the

given) languages and exit.

--show-os Print the value of the operating system mode

and exit. See also --unix, --windows.

-v[=<n>] Verbose switch (optional numeric value).

-verbose[=<n>] Long form of -v.

--version Print the version of this program and exit.

--write-lang-def=<file> Writes to <file> the language processing filters

then exits. Useful as a first step to creating

custom language definitions (see also

--force-lang-def, --read-lang-def).

${BB}Output Options${NN}

--3 Print third-generation language output.

(This option can cause report summation to fail

if some reports were produced with this option

while others were produced without it.)

--by-percent X Instead of comment and blank line counts, show

these values as percentages based on the value

of X in the denominator:

X = 'c' -> # lines of code

X = 'cm' -> # lines of code + comments

X = 'cb' -> # lines of code + blanks

X = 'cmb' -> # lines of code + comments + blanks

For example, if using method 'c' and your code

has twice as many lines of comments as lines

of code, the value in the comment column will

be 200%. The code column remains a line count.

--csv Write the results as comma separated values.

--csv-delimiter=<C> Use the character <C> as the delimiter for comma

separated files instead of ,. This switch forces

--json Write the results as JavaScript Object Notation

(JSON) formatted output.

--md Write the results as Markdown-formatted text.

--out=<file> Synonym for --report-file=<file>.

--progress-rate=<n> Show progress update after every <n> files are

processed (default <n>=100). Set <n> to 0 to

suppress progress output (useful when redirecting

output to STDOUT).

--quiet Suppress all information messages except for

the final report.

--report-file=<file> Write the results to <file> instead of STDOUT.

--sql=<file> Write results as SQL create and insert statements

which can be read by a database program such as

SQLite. If <file> is -, output is sent to STDOUT.

--sql-append Append SQL insert statements to the file specified

by --sql and do not generate table creation

statements. Only valid with the --sql option.

--sql-project=<name> Use <name> as the project identifier for the

current run. Only valid with the --sql option.

--sql-style=<style> Write SQL statements in the given style instead

of the default SQLite format. Currently, the

only style option is Oracle.

--sum-one For plain text reports, show the SUM: output line

even if only one input file is processed.

--xml Write the results in XML.

--xsl=<file> Reference <file> as an XSL stylesheet within

the XML output. If <file> is 1 (numeric one),

writes a default stylesheet, cloc.xsl (or

cloc-diff.xsl if --diff is also given).

This switch forces --xml on.

--yaml Write the results in YAML.

";

# Help information for options not yet implemented:

# --inline Process comments that appear at the end

# of lines containing code.

# --html Create HTML files of each input file showing

# comment and code lines in different colors.

$| = 1; # flush STDOUT

my $start_time = get_time();

my (

$opt_categorized ,

$opt_found ,

@opt_force_lang ,

$opt_lang_no_ext ,

@opt_script_lang ,

$opt_count_diff ,

$opt_diff ,

$opt_diff_alignment ,

$opt_diff_timeout ,

$opt_html ,

$opt_ignored ,

$opt_counted ,

$opt_show_ext ,

$opt_show_lang ,

$opt_progress_rate ,

$opt_print_filter_stages ,

$opt_v ,

$opt_vcs ,

$opt_version ,

$opt_exclude_lang ,

$opt_exclude_list_file ,

$opt_exclude_dir ,

$opt_explain ,

$opt_include_lang ,

$opt_force_lang_def ,

$opt_read_lang_def ,

$opt_write_lang_def ,

$opt_strip_comments ,

$opt_original_dir ,

$opt_quiet ,

$opt_report_file ,

$opt_sdir ,

$opt_sum_reports ,

$opt_processes ,

$opt_unicode ,

$opt_no3 , # accept it but don't use it

$opt_3 ,

$opt_extract_with ,

$opt_by_file ,

$opt_by_file_by_lang ,

$opt_by_percent ,

$opt_xml ,

$opt_xsl ,

$opt_yaml ,

$opt_csv ,

$opt_csv_delimiter ,

$opt_fullpath ,

$opt_json ,

$opt_md ,

$opt_match_f ,

$opt_not_match_f ,

$opt_match_d ,

$opt_not_match_d ,

$opt_skip_uniqueness ,

$opt_list_file ,

$opt_help ,

$opt_skip_win_hidden ,

$opt_read_binary_files ,

$opt_sql ,

$opt_sql_append ,

$opt_sql_project ,

$opt_sql_style ,

$opt_inline ,

$opt_exclude_ext ,

$opt_ignore_whitespace ,

$opt_ignore_case ,

$opt_follow_links ,

$opt_autoconf ,

$opt_sum_one ,

$opt_stdin_name ,

$opt_force_on_windows ,

$opt_force_on_unix , # actually forces !$ON_WINDOWS

$opt_show_os ,

$opt_skip_archive ,

$opt_max_file_size , # in MB

$opt_use_sloccount ,

$opt_no_autogen ,

$opt_force_git ,

);

my $getopt_success = GetOptions(

"by_file|by-file" => \$opt_by_file ,

"by_file_by_lang|by-file-by-lang" => \$opt_by_file_by_lang ,

"categorized=s" => \$opt_categorized ,

"counted=s" => \$opt_counted ,

"include_lang|include-lang=s" => \$opt_include_lang ,

"exclude_lang|exclude-lang=s" => \$opt_exclude_lang ,

"exclude_dir|exclude-dir=s" => \$opt_exclude_dir ,

"exclude_list_file|exclude-list-file=s" => \$opt_exclude_list_file ,

"explain=s" => \$opt_explain ,

"extract_with|extract-with=s" => \$opt_extract_with ,

"found=s" => \$opt_found ,

"count_and_diff|count-and-diff" => \$opt_count_diff ,

"diff" => \$opt_diff ,

"diff-alignment|diff_alignment=s" => \$opt_diff_alignment ,

"diff-timeout|diff_timeout=i" => \$opt_diff_timeout ,

"html" => \$opt_html ,

"ignored=s" => \$opt_ignored ,

"quiet" => \$opt_quiet ,

"force_lang_def|force-lang-def=s" => \$opt_force_lang_def ,

"read_lang_def|read-lang-def=s" => \$opt_read_lang_def ,

"show_ext|show-ext:s" => \$opt_show_ext ,

"show_lang|show-lang:s" => \$opt_show_lang ,

"progress_rate|progress-rate=i" => \$opt_progress_rate ,

"print_filter_stages|print-filter-stages" => \$opt_print_filter_stages ,

"report_file|report-file=s" => \$opt_report_file ,

"out=s" => \$opt_report_file ,

"script_lang|script-lang=s" => \@opt_script_lang ,

"sdir=s" => \$opt_sdir ,

"skip_uniqueness|skip-uniqueness" => \$opt_skip_uniqueness ,

"strip_comments|strip-comments=s" => \$opt_strip_comments ,

"original_dir|original-dir" => \$opt_original_dir ,

"sum_reports|sum-reports" => \$opt_sum_reports ,

"processes=n" => \$opt_processes ,

"unicode" => \$opt_unicode ,

"no3" => \$opt_no3 , # ignored

"3" => \$opt_3 ,

"v|verbose:i" => \$opt_v ,

"vcs=s" => \$opt_vcs ,

"version" => \$opt_version ,

"write_lang_def|write-lang-def=s" => \$opt_write_lang_def ,

"xml" => \$opt_xml ,

"xsl=s" => \$opt_xsl ,

"force_lang|force-lang=s" => \@opt_force_lang ,

"lang_no_ext|lang-no-ext=s" => \$opt_lang_no_ext ,

"yaml" => \$opt_yaml ,

"csv" => \$opt_csv ,

"csv_delimeter|csv-delimiter=s" => \$opt_csv_delimiter ,

"json" => \$opt_json ,

"md" => \$opt_md ,

"fullpath" => \$opt_fullpath ,

"match_f|match-f=s" => \$opt_match_f ,

"not_match_f|not-match-f=s" => \$opt_not_match_f ,

"match_d|match-d=s" => \$opt_match_d ,

"not_match_d|not-match-d=s" => \$opt_not_match_d ,

"list_file|list-file=s" => \$opt_list_file ,

"help" => \$opt_help ,

"skip_win_hidden|skip-win-hidden" => \$opt_skip_win_hidden ,

"read_binary_files|read-binary-files" => \$opt_read_binary_files ,

"sql=s" => \$opt_sql ,

"sql_project|sql-project=s" => \$opt_sql_project ,

"sql_append|sql-append" => \$opt_sql_append ,

"sql_style|sql-style=s" => \$opt_sql_style ,

"inline" => \$opt_inline ,

"exclude_ext|exclude-ext=s" => \$opt_exclude_ext ,

"ignore_whitespace|ignore-whitespace" => \$opt_ignore_whitespace ,

"ignore_case|ignore-case" => \$opt_ignore_case ,

"follow_links|follow-links" => \$opt_follow_links ,

"autoconf" => \$opt_autoconf ,

"sum_one|sum-one" => \$opt_sum_one ,

"by_percent|by-percent=s" => \$opt_by_percent ,

"stdin_name|stdin-name=s" => \$opt_stdin_name ,

"windows" => \$opt_force_on_windows ,

"unix" => \$opt_force_on_unix ,

"show_os|show-os" => \$opt_show_os ,

"skip_archive|skip-archive=s" => \$opt_skip_archive ,

"max_file_size|max-file-size=i" => \$opt_max_file_size ,

"use_sloccount|use-sloccount" => \$opt_use_sloccount ,

"no_autogen|no-autogen" => \$opt_no_autogen ,

"git" => \$opt_force_git ,

);

$opt_by_file = 1 if defined $opt_by_file_by_lang;

my $CLOC_XSL = "cloc.xsl"; # created with --xsl

$CLOC_XSL = "cloc-diff.xsl" if $opt_diff;

die "\n" unless $getopt_success;

print $usage and exit if $opt_help;

my %Exclude_Language = ();

%Exclude_Language = map { $_ => 1 } split(/,/, $opt_exclude_lang)

if $opt_exclude_lang;

my %Exclude_Dir = ();

%Exclude_Dir = map { $_ => 1 } split(/,/, $opt_exclude_dir )

if $opt_exclude_dir ;

die unless exclude_dir_validates(\%Exclude_Dir);

my %Include_Language = ();

%Include_Language = map { $_ => 1 } split(/,/, $opt_include_lang)

if $opt_include_lang;

# Forcibly exclude .svn, .cvs, .hg, .git, .bzr directories. The contents of these

# directories often conflict with files of interest.

$opt_exclude_dir = 1;

$Exclude_Dir{".svn"} = 1;

$Exclude_Dir{".cvs"} = 1;

$Exclude_Dir{".hg"} = 1;

$Exclude_Dir{".git"} = 1;

$Exclude_Dir{".bzr"} = 1;

$Exclude_Dir{".snapshot"} = 1; # NetApp backups

$opt_count_diff = defined $opt_count_diff ? 1 : 0;

$opt_diff = 1 if $opt_diff_alignment;

$opt_exclude_ext = "" unless $opt_exclude_ext;

$opt_ignore_whitespace = 0 unless $opt_ignore_whitespace;

$opt_ignore_case = 0 unless $opt_ignore_case;

$opt_lang_no_ext = 0 unless $opt_lang_no_ext;

$opt_follow_links = 0 unless $opt_follow_links;

$opt_diff_timeout =10 unless $opt_diff_timeout;

$opt_csv = 1 if $opt_csv_delimiter;

$ON_WINDOWS = 1 if $opt_force_on_windows;

$ON_WINDOWS = 0 if $opt_force_on_unix;

$opt_max_file_size = 100 unless $opt_max_file_size;

my $HAVE_SLOCCOUNT_c_count = 0;

if (!$ON_WINDOWS and $opt_use_sloccount) {

# Only bother doing this kludgey test is user explicitly wants

# to use SLOCCount. Debian based systems will hang if just doing

# external_utility_exists("c_count")

# if c_count is in $PATH; c_count expects to have input.

$HAVE_SLOCCOUNT_c_count = external_utility_exists("c_count /bin/sh");

}

if ($opt_use_sloccount) {

if (!$HAVE_SLOCCOUNT_c_count) {

warn "c_count could not be found; ignoring --use-sloccount\n";

$opt_use_sloccount = 0;

} else {

warn "Using c_count, php_count, xml_count, pascal_count from SLOCCount\n";

warn "--diff is disabled with --use-sloccount\n" if $opt_diff;

warn "--count-and-diff is disabled with --use-sloccount\n" if $opt_count_diff;

warn "--unicode is disabled with --use-sloccount\n" if $opt_unicode;

warn "--strip-comments is disabled with --use-sloccount\n" if $opt_strip_comments;

$opt_diff = 0;

$opt_count_diff = undef;

$opt_unicode = 0;

$opt_strip_comments = 0;

}

}

$opt_vcs = 0 if $opt_force_git;

my @COUNT_DIFF_ARGV = undef;

my $COUNT_DIFF_report_file = undef;

if ($opt_count_diff) {

die "--count-and-diff requires two arguments; got ", scalar @ARGV, "\n"

if scalar @ARGV != 2;

# prefix with a dummy term so that $opt_count_diff is the

# index into @COUNT_DIFF_ARGV to work on at each pass

@COUNT_DIFF_ARGV = (undef, $ARGV[0],

$ARGV[1],

[$ARGV[0], $ARGV[1]]); # 3rd pass: diff them

$COUNT_DIFF_report_file = $opt_report_file if $opt_report_file;

}

# Options defaults:

$opt_quiet = 1 if ($opt_md or $opt_json) and !defined $opt_report_file;

$opt_progress_rate = 100 unless defined $opt_progress_rate;

$opt_progress_rate = 0 if defined $opt_quiet;

if (!defined $opt_v) {

$opt_v = 0;

} elsif (!$opt_v) {

$opt_v = 1;

}

if (defined $opt_xsl) {

$opt_xsl = $CLOC_XSL if $opt_xsl eq "1";

$opt_xml = 1;

}

my $skip_generate_report = 0;

$opt_sql_style = 0 unless defined $opt_sql_style;

$opt_sql = 0 unless $opt_sql_style or defined $opt_sql;

if ($opt_sql eq "-" || $opt_sql eq "1") { # stream SQL output to STDOUT

$opt_quiet = 1;

$skip_generate_report = 1;

$opt_by_file = 1;

$opt_sum_reports = 0;

$opt_progress_rate = 0;

} elsif ($opt_sql) { # write SQL output to a file

$opt_by_file = 1;

$skip_generate_report = 1;

$opt_sum_reports = 0;

}

if ($opt_sql_style) {

$opt_sql_style = lc $opt_sql_style;

if (!grep { lc $_ eq $opt_sql_style } qw ( Oracle )) {

die "'$opt_sql_style' is not a recognized SQL style.\n";

}

}

$opt_by_percent = '' unless defined $opt_by_percent;

if ($opt_by_percent and $opt_by_percent !~ m/^(c|cm|cb|cmb)$/i) {

die "--by-percent must be either 'c', 'cm', 'cb', or 'cmb'\n";

}

$opt_by_percent = lc $opt_by_percent;

if (defined $opt_vcs) {

if ($opt_vcs eq "git") {

$opt_vcs = "git ls-files";

my @submodules = invoke_generator('git submodule status');

foreach my $SM (@submodules) {

$SM =~ s/^\s+//; # may have leading space

$SM =~ s/\(\S+\)\s*$//; # may end with something like (heads/master)

my ($checksum, $dir) = split(' ', $SM, 2);

$dir =~ s/\s+$//;

$Exclude_Dir{$dir} = 1;

}

} elsif ($opt_vcs eq "svn") {

$opt_vcs = "svn list -R";

}

}

my $list_no_autogen = 0;

if (defined $opt_no_autogen and scalar @ARGV == 1 and $ARGV[0] eq "list") {

$list_no_autogen = 1;

}

die $brief_usage unless defined $opt_version or

defined $opt_show_lang or

defined $opt_show_ext or

defined $opt_show_os or

defined $opt_write_lang_def or

defined $opt_list_file or

defined $opt_vcs or

defined $opt_xsl or

defined $opt_explain or

$list_no_autogen or

scalar @ARGV >= 1;

die "--diff requires two arguments; got ", scalar @ARGV, "\n"

if $opt_diff and scalar @ARGV != 2;

if ($opt_version) {

printf "$VERSION\n";

exit;

}

replace_git_hash_with_tarfile(\@ARGV);

# 1}}}

# Step 1: Initialize global constants. {{{1

#

my $nFiles_Found = 0; # updated in make_file_list

my (%Language_by_Extension, %Language_by_Script,

%Filters_by_Language, %Not_Code_Extension, %Not_Code_Filename,

%Language_by_File, %Scale_Factor, %Known_Binary_Archives,

%EOL_Continuation_re,

);

my $ALREADY_SHOWED_HEADER = 0;

my $ALREADY_SHOWED_XML_SECTION = 0;

my %Error_Codes = ( 'Unable to read' => -1,

'Neither file nor directory' => -2,

'Diff error (quoted comments?)' => -3,

'Diff error, exceeded timeout' => -4,

'Line count, exceeded timeout' => -5,

);

my @Autogen_to_ignore = no_autogen_files($list_no_autogen);

if ($opt_force_lang_def) {

# replace cloc's definitions

read_lang_def(

$opt_force_lang_def , # Sample values:

\%Language_by_Extension, # Language_by_Extension{f} = 'Fortran 77'

\%Language_by_Script , # Language_by_Script{sh} = 'Bourne Shell'

\%Language_by_File , # Language_by_File{makefile} = 'make'

\%Filters_by_Language , # Filters_by_Language{Bourne Shell}[0] =

# [ 'remove_matches' , '^\s*#' ]

\%Not_Code_Extension , # Not_Code_Extension{jpg} = 1

\%Not_Code_Filename , # Not_Code_Filename{README} = 1

\%Scale_Factor , # Scale_Factor{Perl} = 4.0

\%EOL_Continuation_re , # EOL_Continuation_re{C++} = '\\$'

);

} else {

set_constants( #

\%Language_by_Extension, # Language_by_Extension{f} = 'Fortran 77'

\%Language_by_Script , # Language_by_Script{sh} = 'Bourne Shell'

\%Language_by_File , # Language_by_File{makefile} = 'make'

\%Filters_by_Language , # Filters_by_Language{Bourne Shell}[0] =

# [ 'remove_matches' , '^\s*#' ]

\%Not_Code_Extension , # Not_Code_Extension{jpg} = 1

\%Not_Code_Filename , # Not_Code_Filename{README} = 1

\%Scale_Factor , # Scale_Factor{Perl} = 4.0

\%Known_Binary_Archives, # Known_Binary_Archives{.tar} = 1

\%EOL_Continuation_re , # EOL_Continuation_re{C++} = '\\$'

);

if ($opt_no_autogen) {

foreach my $F (@Autogen_to_ignore) { $Not_Code_Filename{ $F } = 1; }

}

}

if ($opt_read_lang_def) {

# augment cloc's definitions (keep cloc's where there are overlaps)

merge_lang_def(

$opt_read_lang_def , # Sample values:

\%Language_by_Extension, # Language_by_Extension{f} = 'Fortran 77'

\%Language_by_Script , # Language_by_Script{sh} = 'Bourne Shell'

\%Language_by_File , # Language_by_File{makefile} = 'make'

\%Filters_by_Language , # Filters_by_Language{Bourne Shell}[0] =

# [ 'remove_matches' , '^\s*#' ]

\%Not_Code_Extension , # Not_Code_Extension{jpg} = 1

\%Not_Code_Filename , # Not_Code_Filename{README} = 1

\%Scale_Factor , # Scale_Factor{Perl} = 4.0

\%EOL_Continuation_re , # EOL_Continuation_re{C++} = '\\$'

);

}

if ($opt_lang_no_ext and !defined $Filters_by_Language{$opt_lang_no_ext}) {

die_unknown_lang($opt_lang_no_ext, "--lang-no-ext")

}

check_scale_existence(\%Filters_by_Language, \%Language_by_Extension,

\%Scale_Factor);

my $nCounted = 0;

# Process command line provided extension-to-language mapping overrides.

# Make a hash of known languages in lower case for easier matching.

my %Recognized_Language_lc = (); # key = language name in lc, value = true name

foreach my $language (keys %Filters_by_Language) {

my $lang_lc = lc $language;

$Recognized_Language_lc{$lang_lc} = $language;

}

my %Forced_Extension = (); # file name extensions which user wants to count

my $All_One_Language = 0; # set to !0 if --force-lang's <ext> is missing

foreach my $pair (@opt_force_lang) {

my ($lang, $extension) = split(',', $pair);

my $lang_lc = lc $lang;

if (defined $extension) {

$Forced_Extension{$extension} = $lang;

die_unknown_lang($lang, "--force-lang")

unless $Recognized_Language_lc{$lang_lc};

$Language_by_Extension{$extension} = $Recognized_Language_lc{$lang_lc};

} else {

# the scary case--count everything as this language

$All_One_Language = $Recognized_Language_lc{$lang_lc};

}

}

foreach my $pair (@opt_script_lang) {

my ($lang, $script_name) = split(',', $pair);

my $lang_lc = lc $lang;

if (!defined $script_name) {

die "The --script-lang option requires a comma separated pair of ".

"strings.\n";

}

die_unknown_lang($lang, "--script-lang")

unless $Recognized_Language_lc{$lang_lc};

$Language_by_Script{$script_name} = $Recognized_Language_lc{$lang_lc};

}

# If user provided file extensions to ignore, add these to

# the exclusion list.

foreach my $ext (map { $_ => 1 } split(/,/, $opt_exclude_ext ) ) {

$ext = lc $ext if $ON_WINDOWS;

$Not_Code_Extension{$ext} = 1;

}

# If SQL or --by-file output is requested, keep track of directory names

# generated by File::Temp::tempdir and used to temporarily hold the results

# of compressed archives. Contents of the SQL table 't' will be much

# cleaner if these meaningless directory names are stripped from the front

# of files pulled from the archives.

my %TEMP_DIR = ();

my $TEMP_OFF = 0; # Needed for --sdir; keep track of the number of

# scratch directories made in this run to avoid

# file overwrites by multiple extractions to same

# sdir.

# Also track locations where temporary installations, if necessary, of

# Algorithm::Diff and/or Regexp::Common are done. Make sure these

# directories are not counted as inputs (ref bug #80 2012-11-23).

my %TEMP_INST = ();

# invert %Language_by_Script hash to get an easy-to-look-up list of known

# scripting languages

my %Script_Language = map { $_ => 1 } values %Language_by_Script ;

# 1}}}

# Step 2: Early exits for display, summation. {{{1

#

print_extension_info( $opt_show_ext ) if defined $opt_show_ext ;

print_language_info( $opt_show_lang, '') if defined $opt_show_lang;

print_language_filters( $opt_explain ) if defined $opt_explain ;

exit if (defined $opt_show_ext) or

(defined $opt_show_lang) or

(defined $opt_explain) or

$list_no_autogen;

Top_of_Processing_Loop:

# Sorry, coding purists. Using a goto to implement --count-and-diff