=begin
Copyright 2009 to 2017, Andrew Horton

This file is part of WhatWeb.

WhatWeb is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 2 of the License, or
at your option) any later version.

WhatWeb is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with WhatWeb.  If not, see <http://www.gnu.org/licenses/>.
=end

class Output
  # if no f, output to STDOUT,
  # if f is a filename then open it, if f is a file use it
  def initialize(f = STDOUT)
    f = STDOUT if f == "-"
    @f = f if f.class == IO or f.class == File
    @f = File.open(f,"a") if f.class == String
    @f.sync = true # we want flushed output
  end

  def close
    @f.close unless @f.class == IO
  end

  # perform sort, uniq and join on each plugin result
  def suj(plugin_results)
    suj = {}
    [:certainty, :version, :os, :string, :account, :model, :firmware, :module, :filepath].map do |thissymbol|
      t = plugin_results.map { |x| x[thissymbol] unless x[thissymbol].class == Regexp }.flatten.compact.sort.uniq.join(",")
      suj[thissymbol] = t
    end
    suj[:certainty] = plugin_results.map { |x| x[:certainty] }.flatten.compact.sort.last.to_i # this is different, it's a number
    suj
  end


  # sort and uniq but no join. just for one plugin result
  def sortuniq(p)
    su = {}
    [:name, :certainty, :version, :os, :string, :account, :model, :firmware, :module, :filepath].map do |thissymbol|
      unless p[thissymbol].class == Regexp
        t = p[thissymbol]
        t = t.flatten.compact.sort.uniq if t.is_a?(Array)
        su[thissymbol] = t unless t.nil?
      end
    end
    # certainty is different, it's a number
    su[:certainty] = p[:certainty].to_i
    su
  end
end


class OutputObject < Output
  
  def out(target, status, results)
    $semaphore.synchronize do
      @f.puts "Identifying: #{target.to_s}"
      @f.puts "HTTP-Status: #{status.to_s}"
      @f.puts results.pretty_inspect unless results.empty?
      @f.puts
    end
  end
end


class OutputVerbose < Output
  def coloured(s, colour)
    use_colour = ((@f == STDOUT and $use_colour == "auto") or ($use_colour == "always"))
    if use_colour
      send colour, s
    else
      s
    end
  end

  def out(target, status, results)
    $semaphore.synchronize do
      # make a hash of the matches array
      results_hash={}
      results.map { |k, v| results_hash[k] = v }

      display = {
        title: "<None>",
        ip: "<Unknown>",
        country: "<Unknown>",
        status: "<Unknown>"
      }

      display[:country] = results_hash["Country"].map { |r| "#{r[:string]}, #{r[:module]}" }.join(",") if results_hash["Country"]
      display[:ip] = results_hash["IP"].map { |r| r[:string] }.join(",") if results_hash["Country"]
      display[:title] = results_hash["Title"].map { |r| r[:string] }.join(",") if results_hash["Title"]
      display[:status] = "#{status}" + " " + HTTP_Status.code(status)

      @f.puts "WhatWeb report for #{coloured(target,'blue')}"
      @f.puts "Status".ljust(9) + " : "+ display[:status]
      @f.puts "Title".ljust(9) + " : #{coloured(display[:title],'yellow')}"
      @f.puts "IP".ljust(9) + " : " + display[:ip]
      @f.puts "Country".ljust(9) + " : #{coloured(display[:country],'red')}"
      @f.puts

      ################### Short list
      ################### Basically Output Brief

      brief_results = []
      results.each do |plugin_name, plugin_results|
        next if %w[Title IP Country].include? plugin_name

        unless plugin_results.empty?
          suj = suj(plugin_results)

          certainty, version, os, string, accounts, model, firmware, modules, filepath = suj[:certainty].to_i, suj[:version], suj[:os], suj[:string], suj[:account], suj[:model], suj[:firmware], suj[:module], suj[:filepath]

          # colour the output
          # be more DRY
          # if plugins have categories or tags this would be better, eg. all hash plugins are grey
          if (@f == STDOUT and $use_colour == "auto") or ($use_colour == "always")
             coloured_string = grey(string)
             coloured_string = cyan(string) if plugin_name == "HTTPServer"
             coloured_string = yellow(string) if plugin_name == "Title"

             coloured_string = grey(string) if plugin_name == "MD5"
             coloured_string = grey(string) if plugin_name == "Header-Hash"
             coloured_string = grey(string) if plugin_name == "Footer-Hash"
             coloured_string = grey(string) if plugin_name == "CSS"
             coloured_string = grey(string) if plugin_name == "Tag-Hash"

             coloured_plugin = white(plugin_name)
             coloured_plugin = grey(plugin_name) if plugin_name == "MD5"
             coloured_plugin = grey(plugin_name) if plugin_name == "Header-Hash"
             coloured_plugin = grey(plugin_name) if plugin_name == "Footer-Hash"
             coloured_plugin = grey(plugin_name) if plugin_name == "CSS"
             coloured_plugin = grey(plugin_name) if plugin_name == "Tag-Hash"

             p = ((certainty and certainty < 100) ? "#{grey(certainty_to_words(certainty))} " : "")  +
               coloured_plugin + (!version.empty? ? "[#{green(version)}]" : "") +
               (!os.empty? ? "[#{red(os)}]" : "") +
               (!string.empty? ? "[#{coloured_string}]" : "") +
               (!accounts.empty? ? "[#{cyan(accounts)}]" : "" ) +
               (!model.empty? ? "[#{dark_green(model)}]" : "" ) +
               (!firmware.empty? ? "[#{dark_green(firmware)}]" : "" ) +
               (!filepath.empty? ? "[#{dark_green(filepath)}]" : "" ) +
               (!modules.empty? ? "[#{magenta(modules)}]" : "" )

             brief_results << p
          else
            brief_results << ((certainty and certainty < 100) ? "#{certainty_to_words(certainty)} " : "")  +
               plugin_name + (!version.empty? ? "[#{version}]" : "") +
               (!os.empty? ? "[#{os}]" : "") +
               (!string.empty? ? "[#{string}]" : "") +
               (!accounts.empty? ? " [#{accounts}]" : "" ) +
               (!model.empty? ? "[#{model}]" : "" ) +
               (!firmware.empty? ? "[#{firmware}]" : "" ) +
               (!filepath.empty? ? "[#{filepath}]" : "" ) +
               (!modules.empty? ? "[#{modules}]" : "" )
          end
        end
      end

      brief_results_final = brief_results.join(", ")
      @f.puts "Summary".ljust(9) + " : " + brief_results_final
      @f.puts
      @f.puts "Detected Plugins:"

      results.sort.each do |plugin_name, plugin_results|
        next if %w[Title IP Country].include?(plugin_name)
        unless plugin_results.empty?
          @f.puts "[ #{coloured(plugin_name, 'white')} ]"
          description = [""]
          if Plugin.registered_plugins[plugin_name].description
            d = Plugin.registered_plugins[plugin_name].description
            description = word_wrap(d, 60)
          end
  #          @f.puts "\tCategory   : " + Plugin.registered_plugins[plugin_name].category.first unless Plugin.registered_plugins[plugin_name].category.nil?
          @f.puts "\t" + description.first
          description[1..-1].each { |line| @f.puts "\t" + line }
          @f.puts

          top_certainty = suj(plugin_results)[:certainty].to_i
          unless top_certainty == 100
            @f.puts "\t" + "Certainty".ljust(13) + ": " + certainty_to_words(top_certainty)
          end

          plugin_results.map { |x| sortuniq(x) }.each do |pr|
            if pr[:name]
              name_of_match = pr[:name]
            else
              name_of_match = [pr[:regexp_compiled], pr[:text], pr[:regexp].to_s,
                  pr[:ghdb], pr[:md5], pr[:tagpattern]].compact.join("|")
            end

            pr.each do |key,value|
              next unless [:version, :os, :string, :account, :model,
                  :firmware, :module, :filepath, :url].include?(key)

              next if value.class == Regexp

              unless key == :os
                @f.print "\t" + key.to_s.capitalize.ljust(13) + ": "
              else
                @f.print "\t" + "OS".ljust(13) + ": "
              end

              c = case key
                  when :version then "green"
                  when :string then "cyan"
                  when :certainty then "grey"
                  when :os then "red"
                  when :account then "cyan"
                  when :model then "dark_green"
                  when :firmware then "dark_green"
                  when :module then "magenta"
                  when :filepath then "dark_green"
                  else "grey"
                  end

              if value.is_a?(String)
                @f.print coloured(value.to_s,c)
              elsif value.is_a?(Array)
                @f.print coloured(value.join(",").to_s, c)
              else
                @f.print coloured(value.inspect, c)
              end

              unless name_of_match.empty?
                @f.print " (from #{name_of_match})"
              end
              unless pr[:certainty] == 100
                @f.print " (Certainty: #{ ertainty_to_words pr[:certainty]} )"
              end

              @f.puts
            end

            @f.puts "\t" + coloured(pr.inspect.to_s,"dark_blue") if $verbose > 1
          end

          if defined? Plugin.registered_plugins[plugin_name].aggressive
             @f.puts "\tAggressive function available (check plugin file or details)."
          end

          if Plugin.registered_plugins[plugin_name].dorks
            @f.puts "\tGoogle Dorks".ljust(13)+ ": (#{Plugin.registered_plugins[plugin_name].dorks.size})"
          end

          if Plugin.registered_plugins[plugin_name].website
            @f.puts "\tWebsite".ljust(13)+ ": #{Plugin.registered_plugins[plugin_name].website}"
          end

          @f.puts
        end
      end

      @f.puts "HTTP Headers:"
      target.raw_headers.each_line do |header|
        @f.puts "\t#{header}"
        #pp target.raw_headers
      end
    end
  end
end


class OldOutputVerbose < Output
  def coloured(s, colour)
    use_colour = ((@f == STDOUT and $use_colour=="auto") or ($use_colour=="always"))
    if use_colour
      send colour, s
    else
      s
    end
  end

  def out(target, status, results)
    $semaphore.synchronize do
      @f.puts "URL".ljust(7) + ": #{coloured(target,'blue')}"
      @f.puts "Status".ljust(7) + ": #{status}"
      results.sort.each do |plugin_name, plugin_results|
        unless plugin_results.empty?

          @f.puts "   " + coloured(plugin_name, "yellow") + " " + coloured("-"*(80-5-plugin_name.size), "dark_blue")

          description = [""]
          unless Plugin.registered_plugins[plugin_name].description.nil?
            d = Plugin.registered_plugins[plugin_name].description[0..350]
            d += "..." if d.size == 251
            description = word_wrap(d, 60)
          end
#          @f.puts "\tCategory   : " + Plugin.registered_plugins[plugin_name].category.first unless Plugin.registered_plugins[plugin_name].category.nil?

          @f.puts "\tDescription: #{description.first}"
          description[1..-1].each { |line| @f.puts "\t" + " " * 13 + line }

          unless Plugin.registered_plugins[plugin_name].website.nil?
            @f.puts "\tWebsite    : #{Plugin.registered_plugins[plugin_name].website}"
          end

          top_certainty = suj(plugin_results)[:certainty].to_i
          unless top_certainty == 100
            @f.puts "\t" + "Certainty".ljust(11) + ": #{certainty_to_words(top_certainty)}"
          end

          plugin_results.map { |x| sortuniq(x) }.each do |pr|
            if pr[:name]
              name_of_match = pr[:name]
            else
              name_of_match = [pr[:regexp_compiled], pr[:text], pr[:regexp].to_s,
                    pr[:ghdb], pr[:md5], pr[:tagpattern]].compact.join("|")
            end

            pr.each do |key, value|
              next unless [:version, :os, :string, :account, :model,
                  :firmware, :module, :filepath, :url].include?(key)

              next if value.class == Regexp

              @f.print "\t" + key.to_s.capitalize.ljust(11) + ": "

              c = case key
                  when :version then "green"
                  when :string then "cyan"
                  when :certainty then "grey"
                  when :os then "red"
                  when :account then "cyan"
                  when :model then "dark_green"
                  when :firmware then "dark_green"
                  when :module then "magenta"
                  when :filepath then "dark_green"
                  else "grey"
                  end

              if value.is_a?(String)
                @f.print coloured(value.to_s, c)
              elsif value.is_a?(Array)
                @f.print coloured(value.join(",").to_s, c)
              else
                @f.print coloured(value.inspect, c)
              end

              unless name_of_match.empty?
                @f.print " (from #{name_of_match})"
              end
              unless pr[:certainty] == 100
                @f.print " (Certainty: #{certainty_to_words pr[:certainty]} )"
              end

              @f.puts
            end

            @f.puts "\t" + coloured(pr.inspect.to_s, "dark_blue") if $verbose > 1
          end
          @f.puts
        end
      end
    end
  end
end


class OutputBrief < Output

  def escape(s)
    # Encode all special characters # More info: http://www.asciitable.com/
    #r=/[^\x20-\x5A\x5E-\x7E]/

    # Encode low ascii non printable characters
    r = /[\x00-\x1F]/

    # based on code for CGI.escape
    s.gsub(r) { |x| "%#{x.unpack('H2' * x.size).join('%').upcase}" }
  end

  # don't use colours if not to STDOUT
  def out(target, status, results)
    brief_results = []

    # sort results so plugins that are less important at a glance are last
    #    last_plugins=%w| CSS MD5 Header-Hash Footer-Hash Tag-Hash|
    #    results=results.sort_by {|x,y| last_plugins.include?(x) ? 1 : 0 }

    results = results.sort # sort results by plugin name alphabetically

    results.each do |plugin_name, plugin_results|
      unless plugin_results.empty?
        suj = suj(plugin_results)

        certainty, version, os, string, accounts, model, firmware, modules, filepath = suj[:certainty].to_i, escape(suj[:version]), escape(suj[:os]), escape(suj[:string]), escape(suj[:account]), escape(suj[:model]), escape(suj[:firmware]), escape(suj[:module]), escape(suj[:filepath])

        # colour the output
        # be more DRY
        # if plugins have categories or tags this would be better, eg. all hash plugins are grey
        if (@f == STDOUT and $use_colour == "auto") or ($use_colour == "always")
           coloured_string = grey(string)
           coloured_string = cyan(string) if plugin_name == "HTTPServer"
           coloured_string = yellow(string) if plugin_name == "Title"

           coloured_string = grey(string) if plugin_name == "MD5"
           coloured_string = grey(string) if plugin_name == "Header-Hash"
           coloured_string = grey(string) if plugin_name == "Footer-Hash"
           coloured_string = grey(string) if plugin_name == "CSS"
           coloured_string = grey(string) if plugin_name == "Tag-Hash"

           coloured_plugin = white(plugin_name)
           coloured_plugin = grey(plugin_name) if plugin_name == "MD5"
           coloured_plugin = grey(plugin_name) if plugin_name == "Header-Hash"
           coloured_plugin = grey(plugin_name) if plugin_name == "Footer-Hash"
           coloured_plugin = grey(plugin_name) if plugin_name == "CSS"
           coloured_plugin = grey(plugin_name) if plugin_name == "Tag-Hash"

           p = ((certainty and certainty < 100) ? "#{grey(certainty_to_words(certainty))} " : "")  +
             coloured_plugin + (!version.empty? ? "[#{green(version)}]" : "") +
             (!os.empty? ? "[#{red(os)}]" : "") +
             (!string.empty? ? "[#{coloured_string}]" : "") +
             (!accounts.empty? ? "[#{cyan(accounts)}]" : "" ) +
             (!model.empty? ? "[#{dark_green(model)}]" : "" ) +
             (!firmware.empty? ? "[#{dark_green(firmware)}]" : "" ) +
             (!filepath.empty? ? "[#{dark_green(filepath)}]" : "" ) +
             (!modules.empty? ? "[#{red(modules)}]" : "" )

           brief_results << p
        else

          brief_results << ((certainty and certainty < 100) ? "#{certainty_to_words(certainty)} " : "")  +
             plugin_name + (!version.empty? ? "[#{version}]" : "") +
             (!os.empty? ? "[#{os}]" : "") +
             (!string.empty? ? "[#{string}]" : "") +
             (!accounts.empty? ? " [#{accounts}]" : "" ) +
             (!model.empty? ? "[#{model}]" : "" ) +
             (!firmware.empty? ? "[#{firmware}]" : "" ) +
             (!filepath.empty? ? "[#{filepath}]" : "" ) +
             (!modules.empty? ? "[#{modules}]" : "" )
        end
      end
    end

    status_code = HTTP_Status.code(status)

    if (@f == STDOUT and $use_colour=="auto") or ($use_colour=="always")
      brief_results_final = "#{blue(target)} [#{status} #{status_code}] #{brief_results.join(', ')}"
    else
      brief_results_final = "#{target} [#{status} #{status_code}] #{brief_results.join(', ')}"
    end
    $semaphore.synchronize do
      @f.puts brief_results_final
    end
  end
end


# XML Output #
# Does anyone use XML output?
# We'd love to hear any suggestions you may have!
# Does it bother you that some types of output are joined by commas
# but other types aren't?
class OutputXML < Output
  def initialize(f=STDOUT)
    super
    @substitutions = {'&'=>'&amp;', '"'=>'&quot;', '<'=>'&lt;', '>'=>'&gt;'}
    # only output <?xml line if it's a new file or STDOUT
    if @f == STDOUT or @f.size == 0
      @f.puts '<?xml version="1.0"?><?xml-stylesheet type="text/xml" href="whatweb.xsl"?>'
    end
    @f.puts "<log>"
  end

  def close
    @f.puts "</log>"
    @f.close
  end

  def escape(t)
    text = t.to_s.dup
    # use sort_by so that & is before &quot;, etc.
    @substitutions.sort_by { |a, _| a == "&" ? 0 : 1 }.map{ |from, to| text.gsub!(from, to) }

    # Encode all special characters
    # More info: http://www.asciitable.com/
    r = /[^\x20-\x5A\x5E-\x7E]/

    # based on code for CGI.escape
    text.gsub!(r) { |x| "%#{x.unpack('H2' * x.size).join('%').upcase}" }

    text
  end

  def out(target, status, results)
    $semaphore.synchronize do
      @f.puts "<target>"
      @f.puts "\t<uri>#{escape(target)}</uri>"
      @f.puts "\t<http-status>#{escape(status)}</http-status>"

      @f.puts "\t<request-config>"
			@f.puts "\t\t<proxy>" if $USE_PROXY
			@f.puts "\t\t\t<proxy-host>#{escape($PROXY_HOST)}:#{escape($PROXY_PORT)}</proxy-host>" if $PROXY_HOST and $PROXY_PORT
			@f.puts "\t\t\t<proxy-user>#{escape($PROXY_USER)}</proxy-user>" if $PROXY_USER
			@f.puts "\t\t</proxy>" if $USE_PROXY
			$CUSTOM_HEADERS.each do |header_name, header_value|
				@f.puts "\t\t<header>"
				@f.puts "\t\t\t<header-name>#{escape(header_name)}</header-name>"
				@f.puts "\t\t\t<header-value>#{escape(header_value)}</header-value>"
				@f.puts "\t\t</header>"
			end
			@f.puts "\t</request-config>"
      
      results.each do |plugin_name,plugin_results|
        @f.puts "\t<plugin>"
        @f.puts "\t\t<name>#{escape(plugin_name)}</name>"

        unless plugin_results.empty?
          # important info in brief mode is version, type and ?
          # what's the highest probability for the match?

          certainty = plugin_results.map {|x|
            x[:certainty] unless x[:certainty].class == Regexp }.flatten.compact.sort.uniq.last
          version = plugin_results.map {|x|
            x[:version] unless x[:version].class == Regexp }.flatten.compact.sort.uniq
          os = plugin_results.map {|x|
            x[:os] unless x[:os].class == Regexp}.flatten.compact.sort.uniq
          string = plugin_results.map {|x|
            x[:string] unless x[:string].class == Regexp}.flatten.compact.sort.uniq
          model = plugin_results.map {|x|
            x[:model] unless x[:model].class == Regexp}.flatten.compact.sort.uniq
          firmware = plugin_results.map {|x|
            x[:firmware] unless x[:firmware].class == Regexp}.flatten.compact.sort.uniq
          filepath = plugin_results.map {|x|
            x[:filepath] unless x[:filepath].class == Regexp}.flatten.compact.sort.uniq

          account = plugin_results.map {|x|
            x[:account] unless x[:account].class == Regexp}.flatten.compact.sort.uniq
          modules = plugin_results.map {|x|
            x[:module] unless x[:module].class == Regexp}.flatten.compact.sort.uniq

          # Output results
          @f.puts "\t\t<certainty>#{escape(certainty)}</certainty>" if certainty and certainty < 100
          version.map { |x| @f.puts "\t\t<version>#{escape(x)}</version>" }
          os.map { |x| @f.puts "\t\t<os>#{escape(x)}</os>" }
          string.map { |x| @f.puts "\t\t<string>#{escape(x)}</string>" }
          model.map { |x| @f.puts "\t\t<model>#{escape(x)}</model>" }
          firmware.map { |x| @f.puts "\t\t<firmware>#{escape(x)}</firmware>" }
          filepath.map { |x| @f.puts "\t\t<filepath>#{escape(x)}</filepath>" }
          account.map { |x| @f.puts "\t\t<account>#{escape(x)}</account>" }
          modules.map { |x| @f.puts "\t\t<module>#{escape(x)}</module>" }
        end
        @f.puts "\t</plugin>"
      end
      @f.puts "</target>"
    end
  end
end


# MagicTree #
# Output XML file in MagicTree XML format
class OutputMagicTreeXML < Output
  def initialize(f=STDOUT)
    super
    @substitutions={'&'=>'&amp;', '"'=>'&quot;', '<'=>'&lt;', '>'=>'&gt;'}

    # only output <?xml line if it's a new file or STDOUT
    if @f.size == 0
      @f.puts '<?xml version="1.0" encoding="UTF-8"?>'
    end
    @f.puts '<magictree class="MtBranchObject">'
  end

  def close
    @f.puts '</magictree>'
    @f.close
  end

  def escape(t)
    text = t.to_s.dup
    # use sort_by so that & is before &quot;, etc.
    @substitutions.sort_by { |a, _| a == "&" ? 0 : 1 }.map{ |from, to| text.gsub!(from, to) }

    # Encode all special characters
    # More info: http://www.asciitable.com/
    r = /[^\x20-\x5A\x5E-\x7E]/

    # based on code for CGI.escape
    text.gsub!(r) { |x| "%#{x.unpack('H2' * x.size).join('%').upcase}" }
    text
  end

  def out(target, _status, results)
    $semaphore.synchronize do
      # Parse target URL and initialize host node details
      uri = URI.parse(target.to_s)
      @host_os = []
      @host_port = uri.port
      @host_scheme = uri.scheme

      # Set host node details
      if uri.host =~ /^[\d]{1,3}\.[\d]{1,3}\.[\d]{1,3}\.[\d]{1,3}$/i
        @host_ip = uri.host
        @host_name = nil
      else
        @host_name = uri.host
        @host_ip = nil
      end

      # Loop through plugin results # get host IP, country and OS
      results.each do |plugin_name, plugin_results|
        unless plugin_results.empty?
          # Host IP
          @host_ip = plugin_results.map { |x| x[:string] unless x[:string].nil? }.to_s if plugin_name =~ /^IP$/
          # Host Country
          @host_country = plugin_results.map { |x| x[:string] unless x[:string].nil? }.to_s if plugin_name =~ /^Country$/
          # Host OS
          @host_os << plugin_results.map { |x| x[:os] unless x[:os].class == Regexp }.to_s
        end
      end

      # testdata branch
      @f.write "<testdata class=\"MtBranchObject\"><host>#{escape(@host_ip)}"

      # hostname
      @f.write "<hostname>#{escape(@host_name)}</hostname>" unless @host_name.nil?

      # os
      @host_os.compact.sort.uniq.map { |x| @f.write "<os>#{escape(x.to_s)}</os>" unless x.empty? } unless @host_os.empty?

      # country and port nodes
      @f.write "<country>#{escape(@host_country)}</country><ipproto>tcp<port>#{escape(@host_port)}<state>open</state>"

      # https
      if @host_scheme == 'https'
        @f.write "<tunnel>ssl";
      end

      # Service node # Loop through remaining results
      # software, headers, firmware, modules, etc. are all related to a specific URL and therefore are placed under the url node
      @f.puts "<service>http";
      results.each do |plugin_name, plugin_results|
        if !plugin_results.empty? and plugin_name !~ /^IP$/ and plugin_name !~ /^Country$/
          certainty = plugin_results.map { |x| x[:certainty] unless x[:certainty].class == Regexp }.flatten.compact.sort.uniq.last
          versions = plugin_results.map { |x| x[:version] unless x[:version].class == Regexp }.flatten.compact.sort.uniq
          strings = plugin_results.map { |x| x[:string] unless x[:string].class == Regexp }.flatten.compact.sort.uniq
          models = plugin_results.map { |x| x[:model] unless x[:model].class == Regexp }.flatten.compact.sort.uniq
          firmwares = plugin_results.map { |x| x[:firmware] unless x[:firmware].class == Regexp }.flatten.compact.sort.uniq
          filepaths = plugin_results.map { |x| x[:filepath] unless x[:filepath].class == Regexp }.flatten.compact.sort.uniq
          accounts = plugin_results.map { |x| x[:account] unless x[:account].class == Regexp }.flatten.compact.sort.uniq
          modules = plugin_results.map { |x| x[:module] unless x[:module].class == Regexp }.flatten.compact.sort.uniq

          # URL node # plugin node
          @f.write "<url>#{escape(target)}<#{escape(plugin_name)}>"

          # Print certainty if certainty < 100
          if certainty and certainty < 100
            @f.write "<certainty>#{escape(certainty)}</certainty>"
          end

          # Strings
          if strings.size > 0
            strings.map { |x| @f.write escape(x).to_s } unless plugin_name =~ /^IP$/ or plugin_name =~ /^Country$/
          end

          # Versions
          if versions.size > 0
            versions.map { |x| @f.write "<version>#{escape(x)}</version>" }
          end

          # Models
          if models.size > 0
            models.map { |x| @f.puts "<model>#{escape(x)}</model>" }
          end

          # Firmware
          if firmwares.size > 0
            firmwares.map { |x| @f.write "<firmware>#{escape(x)}</firmware>" }
          end

          # Modules
          if modules.size > 0
            modules.map { |x| @f.write "<module>#{escape(x)}</module>" } unless plugin_name =~ /^Country$/
          end

          # Accounts # MagicTree generally uses "user" nodes for account
          if accounts.size > 0
            accounts.map { |x| @f.write "<user>#{escape(x)}</user>" }
          end

          # Local File Filepaths # Not to be confused with file paths in the web root which are returned in Strings
          if filepaths.size > 0
            filepaths.map { |x| @f.write "<filepath>#{escape(x)}</filepath>" }
          end

          # debug node # Uncomment to debug
          # @f.write "<debug title=\"WhatWeb\" class=\"MtTextObject\">Identifying: #{escape(target)}\nHTTP-Status: #{escape(status)}"
          # @f.write "#{escape(results.pretty_inspect)}" unless results.empty?
          # @f.write "</debug>"

          # Close plugin name and URL nodes
          @f.write "</#{escape(plugin_name)}></url>"
        end
      end
      @f.write "</service>";
      # end https node
      if @host_scheme == 'https'
        @f.write "</tunnel>"
      end
      # testdata # close port, host and testdata nodes
      @f.write "</port></ipproto></host></testdata>"
    end
  end
end


# JSON Output #
class OutputJSON < Output
  def initialize(f = STDOUT)
    super
    # opening bracket
    @f.puts "["
  end

  def close
    # empty hash because each hash ends with a comma
    @f.puts "{}"
    # closing bracket
    @f.puts "]"
    @f.close
  end

  def flatten_elements!(obj)
    if obj.class == Hash
      obj.each_value {|x|
        flatten_elements!(x)
      }
    end

    if obj.class == Array
      obj.flatten!
    end
  end

  def utf8_elements!(obj)
    if obj.class == Hash
      obj.each_value { |x| utf8_elements!(x) }
    end

    if obj.class == Array
      obj.each { |x| utf8_elements!(x) }
    end

    if obj.class == String
#      obj=obj.upcase!
#      obj=Iconv.iconv("UTF-8",@charset,obj).join
#pp @charset
#pp obj.encoding
# read this - http://blog.grayproductions.net/articles/ruby_19s_string
      # replace invalid UTF-8 chars
      # based on http://stackoverflow.com/a/8873922/388038
      if String.method_defined?(:encode)
        obj.encode!('UTF-16', 'UTF-8', invalid: :replace, replace: '')
        obj.encode!('UTF-8', 'UTF-16')
      end
      obj.force_encoding('UTF-8')

  #  obj=obj.force_encoding("ASCII-8BIT")
#puts obj.encoding.name
#    obj.encode!("UTF-8",{invalid: :replace,undef: :replace})

    end
  end

  def out(target, status, results)
    # nice

    foo= {:target=>target.to_s, :http_status=>status, :request_config=>{}, :plugins=>{} }

    # request-config
    req_config = {}
    if $USE_PROXY
      req_config[:proxy] = {:proxy_host=>$PROXY_HOST, :proxy_port=>$PROXY_PORT}
      req_config[:proxy][:proxy_user] = $PROXY_USER if $PROXY_USER
    end

    req_config[:headers] = {} unless $CUSTOM_HEADERS.empty?
    $CUSTOM_HEADERS.each do |header_name, header_value|
      req_config[:headers][header_name] = header_value.dup
    end
    foo[:request_config] = req_config

    # plugins
	  
    results.each do |plugin_name,plugin_results|
#      thisplugin = {name: plugin_name}
      thisplugin = {}

      unless plugin_results.empty?
        # important info in brief mode is version, type and ?
        # what's the highest probability for the match?

        certainty = plugin_results.map { |x| x[:certainty] unless x[:certainty].class == Regexp }.flatten.compact.sort.uniq.last

        version = plugin_results.map { |x| x[:version] unless x[:version].class == Regexp }.flatten.compact.sort.uniq
        os = plugin_results.map { |x| x[:os] unless x[:os].class == Regexp }.flatten.compact.sort.uniq
        string = plugin_results.map { |x| x[:string] unless x[:string].class == Regexp }.flatten.compact.sort.uniq
        accounts = plugin_results.map { |x| x[:account] unless x[:account].class == Regexp }.flatten.compact.sort.uniq
        model = plugin_results.map { |x| x[:model] unless x[:model].class == Regexp }.flatten.compact.sort.uniq
        firmware = plugin_results.map { |x| x[:firmware] unless x[:firmware].class == Regexp }.flatten.compact.sort.uniq
        modules = plugin_results.map { |x| x[:module] unless x[:module].class == Regexp }.flatten.compact.sort.uniq
        filepath = plugin_results.map { |x| x[:filepath] unless x[:filepath].class == Regexp }.flatten.compact.sort.uniq

        if !certainty.nil? and certainty != 100
          thisplugin[:certainty] = certainty
        end

        thisplugin[:version] = version unless version.empty?
        thisplugin[:os] = os unless os.empty?
        thisplugin[:string] = string unless string.empty?
        thisplugin[:account] = accounts unless accounts.empty?
        thisplugin[:model] = model unless model.empty?
        thisplugin[:firmware] = firmware unless firmware.empty?
        thisplugin[:module] = modules unless modules.empty?
        thisplugin[:filepath] = filepath unless filepath.empty?
#        foo[:plugins] << thisplugin
        foo[:plugins][plugin_name.to_sym] = thisplugin
      end
    end

    @charset = results.map { |n, r| r[0][:string] if n == "Charset" }.compact.first

    unless @charset.nil? or @charset == "Failed"
      utf8_elements!(foo) # convert foo to utf-8
      flatten_elements!(foo)
    else
      # could not find encoding force UTF-8 anyway
      utf8_elements!(foo)
    end

    $semaphore.synchronize do
      @f.puts JSON::generate(foo) + ","
    end
  end
end

# Elasticseach Output, copy of JSON ouput then a HTTP request to send result to elastic
class OutputElastic < Output

	def initialize(s)
		@host = s[:host] || "127.0.0.1:9200"
		@index = s[:index] || "whatweb"
	end

	def close
		# nothin'
	end

	def flatten_elements!(obj)
		if obj.class == Hash
			obj.each_value {|x|
				flatten_elements!(x)
			}
		end

		if obj.class == Array
			obj.flatten!
		end
	end

	def utf8_elements!(obj)
		if obj.class == Hash
			obj.each_value {|x|
				utf8_elements!(x)
			}
		end

		if obj.class == Array
			obj.each {|x|
				utf8_elements!(x)
			}
		end

		if obj.class == String
#			obj=obj.upcase!
#			obj=Iconv.iconv("UTF-8",@charset,obj).join
#pp @charset
#pp obj.encoding
# read this - http://blog.grayproductions.net/articles/ruby_19s_string
			# replace invalid UTF-8 chars
			# based on http://stackoverflow.com/a/8873922/388038
			if String.method_defined?(:encode)
			  obj.encode!('UTF-16', 'UTF-8', :invalid => :replace, :replace => '')
			  obj.encode!('UTF-8', 'UTF-16')
			end
            obj = obj.force_encoding('UTF-8')

	#	obj=obj.force_encoding("ASCII-8BIT")
#puts obj.encoding.name
#		obj.encode!("UTF-8",{:invalid=>:replace,:undef=>:replace})

		end
	end

	def out(target, status, results)

		# nice | date be like 2009-11-15T14:12:12 to be autodetected by elastic
		foo= {:target=>target.to_s, :http_status=>status, :date=>Time.now.strftime('%FT%T'), :plugins=>{} } 
		
		results.each do |plugin_name,plugin_results|		

#			thisplugin = {:name=>plugin_name}
			thisplugin = {}

			unless plugin_results.empty?
				# important info in brief mode is version, type and ?
				# what's the highest probability for the match?

				certainty = plugin_results.map {|x| x[:certainty] unless x[:certainty].class==Regexp }.flatten.compact.sort.uniq.last

				version = plugin_results.map {|x| x[:version] unless x[:version].class==Regexp }.flatten.compact.sort.uniq
				os = plugin_results.map {|x| x[:os] unless x[:os].class==Regexp }.flatten.compact.sort.uniq
				string = plugin_results.map {|x| x[:string] unless x[:string].class==Regexp }.flatten.compact.sort.uniq
				accounts = plugin_results.map {|x| x[:account] unless x[:account].class==Regexp }.flatten.compact.sort.uniq
				model = plugin_results.map {|x| x[:model] unless x[:model].class==Regexp }.flatten.compact.sort.uniq
				firmware = plugin_results.map {|x| x[:firmware] unless x[:firmware].class==Regexp }.flatten.compact.sort.uniq
				modules = plugin_results.map {|x| x[:module] unless x[:module].class==Regexp }.flatten.compact.sort.uniq
				filepath = plugin_results.map {|x| x[:filepath] unless x[:filepath].class==Regexp }.flatten.compact.sort.uniq

				if !certainty.nil? and certainty != 100
					thisplugin[:certainty] = certainty
				end

				thisplugin[:version] = version unless version.empty?
				thisplugin[:os] = os unless os.empty?
				thisplugin[:string] = string unless string.empty?
				thisplugin[:account] = accounts unless accounts.empty?
				thisplugin[:model] = model unless model.empty?
				thisplugin[:firmware] = firmware unless firmware.empty?
				thisplugin[:module] = modules unless modules.empty?
				thisplugin[:filepath] = filepath unless filepath.empty?
#				foo[:plugins] << thisplugin
				foo[:plugins][plugin_name.to_sym] = thisplugin
			end
		end

		@charset=results.map {|n,r| r[0][:string] if n=="Charset" }.compact.first

		unless @charset.nil? or @charset == "Failed"
			utf8_elements!(foo) # convert foo to utf-8
			flatten_elements!(foo)
		else
			# could not find encoding force UTF-8 anyway
			utf8_elements!(foo)
		end

		url = URI('http://' + @host + '/' + @index + '/whatwebresult')
		req = Net::HTTP::Post.new(url)
		req.body = JSON::generate(foo)
		res = Net::HTTP.start(url.hostname, url.port) {|http|
			http.request(req)
		}

		case res
		when Net::HTTPSuccess
			#ok
		else
			error(res.code + " " + res.message + "\n" + res.body)
		end
	end
end

# basically the same as OutputJSON
class OutputMongo < Output

  def initialize(s)
    host = s[:host] || "0.0.0.0"
    database = s[:database] || raise("Missing MongoDB database name")
    collection = s[:collection] || "whatweb"

    # should make databse and collection comma or fullstop delimited, eg. test,scan

    #@db.authenticate(s[:username], s[:password]) if s[:username]
    options = { :database => database }
    if s[:username]
          options[:username] = s[:username]
          options[:password] = s[:password]
    end

    @db = Mongo::Client.new([ host ], options)
    @coll = @db[collection]
    @charset = nil
  end

  def close
    @db.close
  end

  def flatten_elements!(obj)
    if obj.class == Hash
      obj.each_value {|x|
        flatten_elements!(x)
      }
    end

    if obj.class == Array
      obj.flatten!
    end
  end

  def utf8_elements!(obj)
    if obj.class == Hash
      obj.each_value {|x|
        utf8_elements!(x)
      }
    end

    if obj.class == Array
      obj.each {|x|
        utf8_elements!(x)
      }
    end

    if obj.class == String
#      obj=obj.upcase!
#      obj=Iconv.iconv("UTF-8",@charset,obj).join
      obj.force_encoding('UTF-8')
    end
  end

  def out(target, status, results)
    # nice
   
    foo= {:target=>target.to_s, :http_status=>status, :request_config=>{}, :plugins=>{} }

		# request-config
		req_config = {}
		if $USE_PROXY
			req_config[:proxy] = {:proxy_host=>$PROXY_HOST, :proxy_port=>$PROXY_PORT}
			req_config[:proxy][:proxy_user] = $PROXY_USER if $PROXY_USER
		end

		req_config[:headers] = {} unless $CUSTOM_HEADERS.empty?
		$CUSTOM_HEADERS.each do |header_name, header_value|
			req_config[:headers][header_name] = header_value.dup
		end
    foo[:request_config] = req_config
    
    results.each do |plugin_name, plugin_results|
#      thisplugin = {name: plugin_name}
      thisplugin = {}

      unless plugin_results.empty?
        # important info in brief mode is version, type and ?
        # what's the highest probability for the match?

        certainty = plugin_results.map { |x| x[:certainty] unless x[:certainty].class == Regexp }.compact.sort.uniq.last
        version = plugin_results.map { |x| x[:version] unless x[:version].class == Regexp }.flatten.compact.sort.uniq
        os = plugin_results.map { |x| x[:os] unless x[:os].class == Regexp }.flatten.compact.sort.uniq
        string = plugin_results.map { |x| x[:string] unless x[:string].class == Regexp }.flatten.compact.sort.uniq
        accounts = plugin_results.map { |x| x[:account] unless x[:account].class == Regexp }.flatten.compact.sort.uniq
        model = plugin_results.map { |x| x[:model] unless x[:model].class == Regexp }.flatten.compact.sort.uniq
        firmware = plugin_results.map { |x| x[:firmware] unless x[:firmware].class == Regexp }.flatten.compact.sort.uniq
        modules = plugin_results.map { |x| x[:module] unless x[:module].class == Regexp }.flatten.compact.sort.uniq
        filepath = plugin_results.map { |x| x[:filepath] unless x[:filepath].class == Regexp }.flatten.compact.sort.uniq

        if !certainty.nil? and certainty != 100
          thisplugin[:certainty] = certainty
        end
        thisplugin[:version] = version unless version.empty?
        thisplugin[:os] = os unless os.empty?
        thisplugin[:string] = string unless string.empty?
        thisplugin[:account] = accounts unless accounts.empty?
        thisplugin[:model] = model unless model.empty?
        thisplugin[:firmware] = firmware unless firmware.empty?
        thisplugin[:module] = modules unless modules.empty?
        thisplugin[:filepath] = filepath unless filepath.empty?
#        foo[:plugins] << thisplugin
        foo[:plugins][plugin_name.to_sym] = thisplugin
      end
    end

    @charset = results.map { |n, r| r[0][:string] if n == "Charset" }.compact.first

    unless @charset.nil? or @charset == "Failed"
      utf8_elements!(foo) # convert foo to utf-8
      flatten_elements!(foo)
      @coll.insert_one(foo)
    else
      error("#{target}: Failed to detect Character set and log to MongoDB")
    end

  end
end


# This is not JSON compliant as a list
class OutputJSONVerbose < Output

  def out(target, status, results)
    # brutal and simple
    $semaphore.synchronize do
      @f.puts JSON::fast_generate([target,status,results])
    end
  end

end


class OutputErrors < Output
  # don't need semaphore.synchronize, as it's locked by the error handling routine
  def out(error)
    @f.puts error
  end
end


class OutputSQL < Output
  def flatten_elements!(obj)
    if obj.class == Hash
      obj.each_value { |x| flatten_elements!(x) }
    elsif obj.class == Array
      obj.flatten!
    end
  end

  def escape_for_sql(s)
    s = s.to_s
    if s.nil?
      "''"
    else
      "'"+ s.gsub("'","\'")+"'"
    end
  end

  def initialize(f=STDOUT)
    super
  end

	def create_tables
		# Note that you may encounter the error "1709 - Index column size too large. The maximum column size is 767 bytes." 
		# when using MySQL <= 5.6 with the innodb engine and the utf8mb4 character set
		# 
		# max_hostname_length = 253
		# max_uri_prefix = 10 # covers https://
		# max_url_length = 2048 # old IE limit
		max_target_length = 2048
		
		# feel free to modify this
		@f.puts "CREATE TABLE plugins (plugin_id int NOT NULL AUTO_INCREMENT, name varchar(255) NOT NULL,PRIMARY KEY (plugin_id), UNIQUE (name));"
    	@f.puts "CREATE TABLE targets (target_id int NOT NULL AUTO_INCREMENT, target varchar(#{max_target_length}) NOT NULL, status varchar(10),PRIMARY KEY (target_id), UNIQUE (target, status) );"    
    	@f.puts "CREATE TABLE scans (scan_id int NOT NULL AUTO_INCREMENT, config_id INT NOT NULL, plugin_id INT NOT NULL, target_id INT NOT NULL, version varchar(255), os varchar(255), string varchar(1024), account varchar(1024), model varchar(1024), firmware varchar(1024), module varchar(1024), filepath varchar(1024), certainty varchar(10) ,PRIMARY KEY (scan_id));"
    	@f.puts "CREATE TABLE request_configs (config_id int NOT NULL AUTO_INCREMENT, value TEXT NOT NULL, PRIMARY KEY (config_id) );"
    
    # plugins table
		@f.puts "INSERT INTO plugins (name) VALUES ('Custom-Plugin');"
		@f.puts "INSERT INTO plugins (name) VALUES ('Grep');"

    Plugin::registered_plugins.each do |n, _|
      @f.puts "INSERT INTO plugins (name) VALUES (#{escape_for_sql(n)});"
    end
  end
    

  def out(target, status, results)
    # nice
    foo= {:target=>target, :http_status=>status, :plugins=>{}, :request_config=>{}}

		# config
		req_config = {}
		if $USE_PROXY
			req_config[:proxy] = {:proxy_host=>$PROXY_HOST, :proxy_port=>$PROXY_PORT}
			req_config[:proxy][:proxy_user] = $PROXY_USER if $PROXY_USER
		end

		req_config[:headers] = {} unless $CUSTOM_HEADERS.empty?
		$CUSTOM_HEADERS.each do |header_name, header_value|
			req_config[:headers][header_name] = header_value.dup
		end
		foo[:request_config] = req_config

    results.each do |plugin_name, plugin_results|
      thisplugin = {}

      unless plugin_results.empty?
        # important info in brief mode is version, type and ?
        # what's the highest probability for the match?

        certainty = plugin_results.map { |x| x[:certainty] unless x[:certainty].class == Regexp }.flatten.compact.sort.uniq.last

        version = plugin_results.map { |x| x[:version] unless x[:version].class == Regexp }.flatten.compact.sort.uniq
        os = plugin_results.map {|x| x[:os] unless x[:os].class == Regexp }.flatten.compact.sort.uniq
        string = plugin_results.map { |x| x[:string] unless x[:string].class == Regexp }.flatten.compact.sort.uniq
        accounts = plugin_results.map { |x| x[:account] unless x[:account].class == Regexp }.flatten.compact.sort.uniq
        model = plugin_results.map { |x| x[:model] unless x[:model].class == Regexp }.flatten.compact.sort.uniq
        firmware = plugin_results.map { |x| x[:firmware] unless x[:firmware].class == Regexp }.flatten.compact.sort.uniq
        modules = plugin_results.map { |x| x[:module] unless x[:module].class == Regexp }.flatten.compact.sort.uniq
        filepath = plugin_results.map { |x| x[:filepath] unless x[:filepath].class == Regexp }.flatten.compact.sort.uniq

        thisplugin[:certainty] = certainty if !certainty.nil? and certainty != 100
        # empty arrays
        thisplugin[:version] = version.empty? ? [] : version
        thisplugin[:os] = os.empty? ? [] : os
        thisplugin[:string] = string.empty? ? [] : string
        thisplugin[:account] = accounts.empty? ? [] : accounts
        thisplugin[:model] = model.empty? ? [] : model
        thisplugin[:firmware] = firmware.empty? ? [] : firmware
        thisplugin[:module] = modules.empty? ? [] : modules
        thisplugin[:filepath] = filepath.empty? ? [] : filepath

        foo[:plugins][plugin_name.to_sym] = thisplugin
      end
    end

    flatten_elements!(foo)

    i_target = escape_for_sql(foo[:target])
    insert = [escape_for_sql(foo[:http_status]), i_target].join(",")
    query = "INSERT IGNORE INTO targets (status,target) VALUES (#{ insert });";
    @f.puts query
   
		insert = [escape_for_sql(JSON.dump(foo[:request_config]))].join(",")
		query = "INSERT INTO request_configs (value) VALUES (#{insert})"
		@f.puts query
    
    foo[:plugins].each do |x|
      plugin_name = escape_for_sql(x.first.to_s)
      insert = [escape_for_sql(x[1][:version].join(",").to_s), escape_for_sql(x[1][:os].join(",").to_s),
          escape_for_sql(x[1][:string].join(",").to_s),   escape_for_sql(x[1][:account].join(",").to_s), escape_for_sql(x[1][:model].join(",").to_s),
          escape_for_sql(x[1][:firmware].join(",").to_s),  escape_for_sql(x[1][:module].join(",").to_s), escape_for_sql(x[1][:filepath].join(",").to_s),
          escape_for_sql(x[1][:certainty].to_s)].join(",")

      query = "INSERT INTO scans (target_id, config_id, plugin_id, version, os, string, account, model, firmware, module, filepath, certainty) VALUES ( (SELECT target_id from targets WHERE target = #{i_target}),(SELECT MAX(config_id) from request_configs),(SELECT plugin_id from plugins WHERE name = #{plugin_name}), #{insert} );";
      @f.puts query
    end
  end
end
                    
        
