#!/usr/bin/ruby
# vim: et sw=2

VER="(3eb9bfb)"
ME="blgrep: "
$col="1;36"
$rs=78

$csv=false
$dups=false
$wib=false
$im=false
$sqs=false
$cnt=false
$dwb=false
$dbh=false
$dbm=false
$sort=nil
$sortmode=nil
$incl=[]
$excl=[]
$catl=[]
$inclj=[]
$exclj=[]
$by=nil
$fld=nil
$fldm=nil
$chk=nil
$join=nil
$expr=[]
$aggr=nil
$db={}
$ma=[]

def in_list(s,list)
  for i in list
    if i.match(s)
      return true
    end
  end
  return false
end

def regexfix(str)
  tmp=[]
  for t in str.scan(/(_)|(\|)|([^_\|]*)/).flatten.compact
    tmp.push(t) if not t.empty?
  end
  res=""
  c=0
  while t=tmp.shift
    c+=1
    if t=="|"
      res+="|"
      c=0
      next
    end
    if t=="_"
      if tmp.empty? or tmp[0]=="|"
        res+="([ \t.:()<>]+|$)"
      elsif c==1
        res+="([ \t.:()<>]+|^)"
      else
        res+="[ \t.:()<>]+"
      end
      next
    end
    res+="#{t}"
  end
  return res
end

def regexfix2(str)
  tmp=[]
  for t in str.scan(/(_)|(\|)|([^_\|]*)/).flatten.compact
    tmp.push(t) if not t.empty?
  end
  res=""
  while t=tmp.shift
    case t
      when("_") then res+=(tmp.empty? or tmp[0]=="|") ? '([ ."\[\]\(\)\{\}]+|$)' : '[ ."\[\]\(\)\{\}]+'
      when("|") then res+="|"
      else res+="#{t}"
    end
  end
  return res
end

def scomp(x,y)
  return 0 if x.empty? and y.empty?
  return -1 if x.empty?
  return 1 if y.empty?
  xa=x.scan(/([0-9]+)|([^0-9]+)/)
  ya=y.scan(/([0-9]+)|([^0-9]+)/)
  r=nil
  xi=xa.shift
  yi=ya.shift
  while xi and yi
    if xi[0] and yi[0]
      xv=xi[0].to_i
      yv=yi[0].to_i
      if(xv==yv)
        r=0
        xi=xa.shift
        yi=ya.shift
        next
      end
      r=(xv>yv) ? 1 : -1
      break
    end
    if xi[1] and yi[1]
      xv=xi[1]
      yv=yi[1]
      if(xv==yv)
        r=0
        xi=xa.shift
        yi=ya.shift
        next
      end
      r=(xv<=>yv)
      break
    end
    break
  end
  return (x<=>y) if !r
  if r==0
    r=-1 if !xi and yi
    r=1 if xi and !yi
  end
  return r
end

$output_cnt=0
class Expr
  FN0=["count","show"]
  FN1=["count","sum","min","max","avg","add","sub","icnt","show","pfxl","pfxc"]
  FN2=["sumif"]
  FNS=FN1+FN2
  AGG=["count","sum","sumif","min","max","avg"]
  SAG=["add","sub"]
  OPS=["+","-","*","/"]
  CMP=["=","==","!=","<","<=",">",">="]
  LOG=["|","&","^","?","!"]
  ALL=OPS+CMP+LOG
  OPL={
    "|"=>1,"&"=>1,"^"=>1,
    "="=>2,"=="=>2,"!="=>2,"<"=>2,"<="=>2,">"=>2,">="=>2,
    "+"=>3,"-"=>3,"*"=>4,"/"=>4,
    "?"=>0,"!"=>9
  }

  Ops=Struct.new :ops, :op, :ac

  attr_reader :fld, :aggr

  def initialize(arr)
    @aggr=0 # 1:AGG, 2:SAG, 4:vaggr
    @show=false
    @temp=false
    @fld=nil
    @str=arr.join
    @ops=[]
    tmp=[]
    tmp[0]=Ops.new([],nil,0)
    i=0
    if arr.length>1 and arr[1]==":"
      t=arr.shift
      if m=/^#(.*)/.match(t)
        @fld=m[1]
        @temp=true
      else
        @fld=t
      end
      arr.shift
    end
    for s in arr
      if s=="," or s==")"
        while i>0 and tmp[i].ac==0
          tmp[i].ops.push(tmp[i].op) if tmp[i].op
          tmp[i-1].ops+=tmp[i].ops
          i-=1
        end
        if tmp[i].op=="?"
          if tmp[i].ac!=-3
            $stderr.puts ME+"syntax error"
            exit
          end
          tmp[i].ops.push(tmp[i].op)
          tmp[i-1].ops+=tmp[i].ops
          i-=1
        end
      end
      if s==":"
        while i>0 and tmp[i].ac==0
          tmp[i].ops.push(tmp[i].op) if tmp[i].op
          tmp[i-1].ops+=tmp[i].ops
          i-=1
        end
        if tmp[i].op!="?" or tmp[i].ac!=-2
          $stderr.puts ME+"syntax error"
          exit
        end
        tmp[i].ac=-3
        next
      end
      if s=="?"
        if tmp[i].op=="?"
          $stderr.puts ME+"syntax error"
          exit
        end
        if tmp[i].op
          tmp[i].ops.push(tmp[i].op)
          tmp[i].op=nil
        end
        tmp[i+1]=Ops.new([],s,-2)
        i+=1
        next
      end
      if s=="()"
        if !tmp[i].op or not FNS.include?(tmp[i].op)
          $stderr.puts ME+"syntax error"
          exit
        end
        if not FN0.include?(tmp[i].op)
          $stderr.puts ME+"function needs argument"
          exit
        end
        tmp[i].ops.push("1",tmp[i].op)
        tmp[i].op=nil
        next
      end
      if s==","
        if i<1 or tmp[i-1].ac<2
          $stderr.puts ME+"syntax error"
          exit
        end
        tmp[i].ops.push(tmp[i].op) if tmp[i].op
        tmp[i].op=nil
        tmp[i-1].ac-=1
        next
      end
      if s=="("
        tmp[i+1]=Ops.new([],nil,-1)
        i+=1
        next
      end
      if s==")"
        if tmp[i].ac!=-1
          $stderr.puts ME+"parantheses mismatch"
          exit
        end
        tmp[i].ops.push(tmp[i].op) if tmp[i].op
        tmp[i-1].ops+=tmp[i].ops
        i-=1
        next if tmp[i].ac<1
        if tmp[i].ac>1
          $stderr.puts ME+"missing argument"
          exit
        end
        if i<1
          tmp[i].ops.push(tmp[i].op) if tmp[i].op
          tmp[i].op=nil
          tmp[i].ac=0
          next
        end
        tmp[i].ops.push(tmp[i].op) if tmp[i].op
        tmp[i-1].ops+=tmp[i].ops
        i-=1
        next
      end
      if s=="!"
        if !tmp[i].op and tmp[i].ac==0
          tmp[i].op=s
          next
        end
        tmp[i+1]=Ops.new([],s,0)
        i+=1
        next
      end
      if FNS.include?(s)
        @aggr|=1 if AGG.include?(s)
        @aggr|=2 if SAG.include?(s)
        if ("%b" % @aggr).count("1")>1
          $stderr.puts ME+"cannot mix aggregation functions"
          exit
        end
        @show=true if s=="show"
        c=0
        c=1 if FN1.include?(s)
        c=2 if FN2.include?(s)
        if !tmp[i].op and tmp[i].ac==0
          tmp[i].op=s
          tmp[i].ac=c
          next
        end
        tmp[i+1]=Ops.new([],s,c)
        i+=1
        next
      end
      if ALL.include?(s)
        if !tmp[i].op
          tmp[i].op=s
          next
        end
        if OPL[s]>OPL[tmp[i].op]
          tmp[i+1]=Ops.new([tmp[i].ops.pop],s,0)
          i+=1
          next
        end
        if OPL[s]==OPL[tmp[i].op]
          tmp[i].ops.push(tmp[i].op)
          tmp[i].op=s
          next
        end
        while i>0 and tmp[i].ac==0
          tmp[i].ops.push(tmp[i].op) if tmp[i].op
          tmp[i-1].ops+=tmp[i].ops
          i-=1
        end
        tmp[i].ops.push(tmp[i].op)
        tmp[i].op=s
        next
      end
      tmp[i].ops.push(s)
      if tmp[i].op=="!"
        tmp[i].ops.push(tmp[i].op)
        tmp[i-1].ops+=tmp[i].ops
        i-=1
      end
    end
    if tmp[i].ac==-1
      $stderr.puts ME+"parantheses mismatch"
      exit
    end
    while i>1 and tmp[i].ac==0
      tmp[i].ops.push(tmp[i].op) if tmp[i].op
      tmp[i-1].ops+=tmp[i].ops
      i-=1
    end
    if tmp[i].op=="?"
      tmp[i].ops.push(tmp[i].op)
      tmp[i-1].ops+=tmp[i].ops
      i-=1
    end
    if i>1
      $stderr.puts ME+"stack error"
      exit
    end
    if i==1
      tmp[i].ops.push(tmp[i].op) if tmp[i].op
      tmp[i-1].ops+=tmp[i].ops
    end
    @ops=tmp[0].ops
    @ops.push(tmp[0].op) if tmp[0].op
  end

  def self.parse(str)
    expr=[]
    arr=[]
    pc=0
    for s in str.scan(/!=|==|<=|>=|\(\)|[\:\,\=\?\<\>\+\-\*\/\!\|\&\^\(\)]|"[^"]*"|[@#a-zA-Z 0-9\.]*/)
      next if s.empty?
      if s=="," and pc==0
        e=Expr.new(arr)
        e.chk_vaggr(expr)
        expr<<e
        arr=[]
        next
      end
      pc+=1 if s=="("
      pc-=1 if s==")"
      arr<<s
    end
    if not arr.empty?
      e=Expr.new(arr)
      e.chk_vaggr(expr)
      expr<<e
    end
    return expr
  end

  def exec(fld,av,vars,bl=false)
    s=[]
    ar=[]
    for o in @ops
      if o=="count"
        if s.length<1
          $stderr.puts ME+"expr: count(): need argument"
          exit
        end
        a=s.pop
        if !a
          r=av.shift
          r=0 if !r
          s.push(r)
          ar.push(r)
          next
        end
        if a.class==String
          v=(a.empty? or a=="-") ? 0 : 1
          r=av.shift
          r=0 if !r
          v=v+r
          s.push(v)
          ar.push(v)
          next
        end
        a=a.to_i
        v=(a==0) ? 0 : 1
        r=av.shift
        r=0 if !r
        v=v+r
        s.push(v)
        ar.push(v)
        next
      end
      if o=="sum"
        if s.length<1
          $stderr.puts ME+"expr: sum(): need argument"
          exit
        end
        a=s.pop
        if !a
          $stderr.puts ME+"expr: sum(): invalid argument"
          exit
        end
        v=a.to_f
        r=av.shift
        r=0.0 if !r
        v=v+r
        s.push(v)
        ar.push(v)
        next
      end
      if o=="sumif"
        if s.length<2
          $stderr.puts ME+"expr: sumif(): need 2 arguments"
          exit
        end
        b=s.pop
        a=s.pop
        if !b or !a
          $stderr.puts ME+"expr: sumif(): invalid arguments"
          exit
        end
        if b.class==String
          b=(b.empty? or b=="-") ? 0 : 1
        else
          b=b.to_i
        end
        v=(b==0) ? 0.0 : a.to_f
        r=av.shift
        r=0.0 if !r
        v=v+r
        s.push(v)
        ar.push(v)
        next
      end
      if o=="min"
        if s.length<1
          $stderr.puts ME+"expr: min(): need argument"
          exit
        end
        a=s.pop
        if !a
          $stderr.puts ME+"expr: min(): invalid argument"
          exit
        end
        v=a.to_f
        r=av.shift
        v=(r<v) ? r : v if r
        s.push(v)
        ar.push(v)
        next
      end
      if o=="max"
        if s.length<1
          $stderr.puts ME+"expr: max(): need argument"
          exit
        end
        a=s.pop
        if !a
          $stderr.puts ME+"expr: max(): invalid argument"
          exit
        end
        v=a.to_f
        r=av.shift
        v=(r>v) ? r : v if r
        s.push(v)
        ar.push(v)
        next
      end
      if o=="avg"
        if s.length<1
          $stderr.puts ME+"expr: avg(): need argument"
          exit
        end
        a=s.pop
        if !a
          $stderr.puts ME+"expr: avg(): invalid argument"
          exit
        end
        v=a.to_f
        rs=av.shift
        rc=av.shift
        if rs and rc
          rs+=v
          rc+=1
          v=rs/rc
        else
          rs=v
          rc=1
        end
        s.push(v)
        ar.push(rs,rc)
        next
      end
      if o=="add"
        if s.length<1
          $stderr.puts ME+"expr: add(): need argument"
          exit
        end
        a=s.pop
        if !a
          $stderr.puts ME+"expr: add(): invalid argument"
          exit
        end
        v=a.to_f
        r=av.shift
        r=0.0 if !r
        v=(bl) ? r+v : 0.0
        s.push(v)
        ar.push(v)
        next
      end
      if o=="sub"
        if s.length<1
          $stderr.puts ME+"expr: sub(): need argument"
          exit
        end
        a=s.pop
        if !a
          $stderr.puts ME+"expr: sub(): invalid argument"
          exit
        end
        v=a.to_f
        r=av.shift
        r=0.0 if !r
        v=(bl) ? r-v : v
        s.push(v)
        ar.push(v)
        next
      end
      if o=="icnt"
        if s.length<1
          $stderr.puts ME+"expr: icnt(): need argument"
          exit
        end
        a=s.pop
        if !a
          $stderr.puts ME+"expr: icnt(): invalid argument"
          exit
        end
        v=a.split(/[, ]+/).compact.length
        s.push(v)
        next
      end
      if o=="show"
        if s.length<1
          $stderr.puts ME+"expr: show(): need argument"
          exit
        end
        a=s.pop
        if !a
          s.push(0)
          next
        end
        if a.class==String
          a=(a.empty? or a=="-") ? 0 : 1
        else
          a=a.to_i
        end
        v=(a==0) ? 0 : 1
        s.push(v)
        next
      end
      if o=="pfxl"
        if s.length<1
          $stderr.puts ME+"expr: pfxl(): need argument"
          exit
        end
        a=s.pop
        if !a or a.class!=String
          $stderr.puts ME+"expr: pfxl(): invalid argument"
          next
        end
        d=(a.include?(":")) ? 128 : 32
        t=a.split("/")
        if t.length!=2
          s.push(d)
          next
        end
        v=t[1].to_i
        s.push(v)
        next
      end
      if o=="pfxc"
        if s.length<1
          $stderr.puts ME+"expr: pfxc(): need argument"
          exit
        end
        a=s.pop
        if !a or a.class!=String
          $stderr.puts ME+"expr: pfxc(): invalid argument"
          next
        end
        if a.include?(":")
          s.push(0)
          next
        end
        t=a.split("/")
        if t.length==2
          v=32-t[1].to_i
          s.push(1<<v)
          next
        end
        t=a.split(".")
        if t.length!=4
          s.push(0)
          next
        end
        b=t[3]
        t=b.split("-")
        if t.length!=2
          s.push(1)
          next
        end
        v=t[1].to_i-t[0].to_i
        s.push(v+1)
        next
      end
      if o=="!"
        a=s.pop
        if a.class==String
          a=(a.empty? or a=="-") ? 0 : 1
        else
          a=a.to_i
        end
        v=(a==0) ? 1 : 0
        s.push(v)
        next
      end
      if o=="?"
        c=s.pop
        b=s.pop
        a=s.pop
        if a.class==String
          a=(a.empty? or a=="-") ? 0 : 1
        else
          a=a.to_i
        end
        v=(a==0) ? c : b
        s.push(v)
        next
      end
      if o=="|"
        b=s.pop
        if b.class==String
          b=(b.empty? or b=="-") ? 0 : 1
        else
          b=b.to_i
        end
        a=s.pop
        if a.class==String
          a=(a.empty? or a=="-") ? 0 : 1
        else
          a=a.to_i
        end
        v=a|b
        s.push(v)
        next
      end
      if o=="&"
        b=s.pop
        if b.class==String
          b=(b.empty? or b=="-") ? 0 : 1
        else
          b=b.to_i
        end
        a=s.pop
        if a.class==String
          a=(a.empty? or a=="-") ? 0 : 1
        else
          a=a.to_i
        end
        v=a&b
        s.push(v)
        next
      end
      if o=="^"
        b=s.pop
        if b.class==String
          b=(b.empty? or b=="-") ? 0 : 1
        else
          b=b.to_i
        end
        a=s.pop
        if a.class==String
          a=(a.empty? or a=="-") ? 0 : 1
        else
          a=a.to_i
        end
        v=a^b
        s.push(v)
        next
      end
      if o=="+"
        b=s.pop.to_f
        a=s.pop.to_f
        v=a+b
        s.push(v)
        next
      end
      if o=="-"
        b=s.pop.to_f
        a=s.pop.to_f
        v=a-b
        s.push(v)
        next
      end
      if o=="*"
        b=s.pop.to_f
        a=s.pop.to_f
        v=a*b
        s.push(v)
        next
      end
      if o=="/"
        b=s.pop.to_f
        a=s.pop.to_f
        v=a/b
        s.push(v)
        next
      end
      if o=="=" or o=="=="
        b=s.pop
        if b.class==Fixnum
          a=s.pop.to_i
          v=(a==b) ? 1 : 0
        elsif b.class==Float
          a=s.pop.to_f
          v=(a==b) ? 1 : 0
        else 
          a=s.pop.downcase
          v=(a.include?(b.downcase)) ? 1 : 0
        end
        s.push(v)
        next
      end
      if o=="!="
        b=s.pop
        if b.class==Fixnum
          a=s.pop.to_i
          v=(a!=b) ? 1 : 0
        elsif b.class==Float
          a=s.pop.to_i
          v=(a!=b) ? 1 : 0
        else
          a=s.pop.downcase
          v=(a.include?(b.downcase)) ? 0 : 1
        end
        s.push(v)
        next
      end
      if o=="<"
        b=s.pop.to_f
        a=s.pop.to_f
        v=(a<b) ? 1 : 0
        s.push(v)
        next
      end
      if o=="<="
        b=s.pop.to_f
        a=s.pop.to_f
        v=(a<=b) ? 1 : 0
        s.push(v)
        next
      end
      if o==">"
        b=s.pop.to_f
        a=s.pop.to_f
        v=(a>b) ? 1 : 0
        s.push(v)
        next
      end
      if o==">="
        b=s.pop.to_f
        a=s.pop.to_f
        v=(a>=b) ? 1 : 0
        s.push(v)
        next
      end
      if /^"/.match(o)
        o=o.delete('"')
        s.push(o)
        next
      end
      if /^[0-9.]+$/.match(o)
        if o.include?(".")
          o=o.to_f
        else
          o=o.to_i
        end
        s.push(o)
        next
      end
      t=o.downcase
      if m=/^@(.*)/.match(t)
        if bl
          o="" if !o=vars["@"+m[1]]
        else
          o="" if !o=fld[m[1]]
          vars["@"+m[1]]=o
        end
      elsif fld.has_key?(t)
        o="" if !o=fld[t]
      elsif vars.has_key?(t)
        o="" if !o=vars[t]
      else
        $stderr.puts ME+"expr: unknown var: "+o
        exit
      end
      s.push(o)
    end
    if not av.empty?
      $stderr.puts ME+"syntax error"
      exit
    end
    av.replace(ar)
    v=s.pop
    vars[@fld]=v if @fld
    return v
  end

  def name
    return @fld if @fld
    return @str
  end

  def temp?
    return @temp
  end

  def aggr?(*ba)
    for b in ba
      return true if (@aggr&b)==b
    end
    return false
  end

  def show?
    return @show
  end

  def chk_vaggr(expr)
    vars=[]
    for o in @ops
      next if FNS.include?(o) or ALL.include?(o)
      vars<<o
    end
    $expr=[] if !$expr
    for e in expr+$expr
      next if !e.fld
      next if not e.aggr?(2)
      if vars.include?(e.fld)
        @aggr|=4
        return
      end
    end
  end
end

$lst_fields=["vlan","vpi","vci","as","slot"]
$ip_fields=["ip"]
$ifn_fields=["link","host"]
$id_fields=["asset","asset num","4budget id","order id"]
$all_fields=$lst_fields+$ip_fields+$id_fields

class Object
  def _str(p=nil)
    return to_s if !p or self.class!=Float
    return "%.#{p}f" % self
  end
end

class Aggr
  def initialize()
    @cnt=0
    @agr=nil
  end

  def init()
    @agr={}
  end

  def add_cnt(cnt)
    @cnt+=cnt
  end

  def add_agr(av)
    for e in $expr
      next if not e.aggr?(1)
      @agr[e]=0 if !@agr[e]
      @agr[e]+=av[e]
    end
  end
end

def get_fields(fld)
  fields=[]
  ft=[0]
  tmp=fld.downcase.split(/\t+/)
  tmp.each { |t| fields.push(t) }
  tmp=fld.scan(/[^\t]*\t*/)
  tmp.pop
  tc=0
  for t in tmp
    tc+=t.count("\t")+t.count("^\t")/8
    ft<<tc
  end
  return fields,ft
end

class FldMatch
  attr_reader :im

  def match(h)
    return false if !vs=h[@fld]
    vs=vs.downcase
    if @type=="lst"
      return false if vs=="-" or vs=="?"
      return false if vs.start_with?("-")
      va=rexpand(vs)
      f=true
      for m in @mas
        if not va.include?(m)
          f=false
        else
          return true if not @em
        end
      end
      return f
    end
    if @type=="ip"
      return false if vs=="-" or vs=="?"
      return true if net_match2(@mas,vs)
      return false
    end
    if @type=="ifn"
      if @em
        vs=vs.split(":")[0]
        return true if vs==@mas
      else
        return true if @mas.match(vs)
      end
      return false
    end
    if @type=="id"
      return false if vs=="-"
      if @mas.include?("-")
        return true if vs==@mas
        return false
      end
      vs=vs.split("-")[0]
      return true if vs==@mas.split("-")[0]
      return false
    end
    if @type=="str"
      if @em
        return true if vs==@mas
      else
        return true if @mas.match(vs)
      end
      return false
    end
    return false
  end

  def chk_fld(fields)
    t=nil
    for k in fields
      next if k.class!=String
      if k.start_with?(@sfld)
        return false if t
        t=k
      end
    end
    @fld=t
    return (t) ? true : false
  end
  
  private

  def rexpand(s)
    tmp=[]
    for v in s.split(",")
      v.slice!("+") if v.start_with?("+")
      v.slice!("*") if v.start_with?("*")
      if v.include?("-")
        t=v.split("-")
        t[0].to_i.upto(t[1].to_i) { |i| tmp+=[i.to_s] }
      else
        tmp+=[v]
      end
    end
    return tmp
  end

  def net_match2(si,mi)
    return net6_match2(si,mi) if si.include?(":") or mi.include?(":")
    tmp=si.split("/")
    sa=tmp[0].split(".")
    sp=(tmp[1]) ? tmp[1].to_i : 32
    tmp=mi.split("/")
    ma=tmp[0].split(".")
    mp=(tmp[1]) ? tmp[1].to_i : 32
    while sp>7 and mp>7
      m=m1=m2=nil
      s=s1=s2=nil
      tmp=ma.shift.split("-")
      if tmp[1]
        m1=tmp[0].to_i
        m2=tmp[1].to_i
        m=nil
      else
        m=tmp[0].to_i
      end
      f=false
      for t in sa.shift.split(",")
        if t=="*"
          f=true
          next
        end
        tmp=t.split("-")
        if tmp[1]
          s1=tmp[0].to_i
          s2=tmp[1].to_i
          s=nil
        else
          s=tmp[0].to_i
        end
        f=true if net_chk(s,m,s1,s2,m1,m2)
      end
      return false if not f
      sp-=8
      mp-=8
    end
    return false if sp<mp
    return true if mp==0
    bf=8-mp
    bf=255<<bf
    m=m1=m2=nil
    s=s1=s2=nil
    tmp=ma.shift.split("-")
    if tmp[1]
      m1=tmp[0].to_i&bf
      m2=tmp[1].to_i&bf
      m=nil
    else
      m=tmp[0].to_i&bf
    end
    for t in sa.shift.split(",")
      if t=="*"
        return true
      end
      tmp=t.split("-")
      if tmp[1]
        s1=tmp[0].to_i&bf
        s2=tmp[1].to_i&bf
        s=nil
      else
        s=tmp[0].to_i&bf
      end
      return true if net_chk(s,m,s1,s2,m1,m2)
    end
    return false
  end

  def net6_match2(si,mi)
    tmp=si.split("/")
    sa=tmp[0].split(":")
    sp=(tmp[1]) ? tmp[1].to_i : 128
    sc=sa.length
    sz=0
    tmp=mi.split("/")
    ma=tmp[0].split(":")
    mp=(tmp[1]) ? tmp[1].to_i : 128
    mc=ma.length
    mz=0
    while sp>15 and mp>15
      m=m1=m2=nil
      s=s1=s2=nil
      if mz>0
        m=0
        mz-=1
      else
        mt=ma.shift
        if !mt
          m=0
          mz=8-mc-1
        elsif mt.empty?
          m=0
          mz=8-mc
        else
          tmp=mt.split("-")
          if tmp[1]
            m1=tmp[0].to_i(16)
            m2=tmp[1].to_i(16)
            m=nil
          else
            m=tmp[0].to_i(16)
          end
        end
      end
      f=false
      if sz>0
        s=0
        sz-=1
        f=true if net_chk(s,m,s1,s2,m1,m2)
      else
        st=sa.shift
        if !st
          s=0
          sz=8-sc-1
          f=true if net_chk(s,m,s1,s2,m1,m2)
        elsif st.empty?
          s=0
          sz=8-sc
          f=true if net_chk(s,m,s1,s2,m1,m2)
        else
          for t in st.split(",")
            if t=="*"
              f=true
              next
            end
            tmp=t.split("-")
            if tmp[1]
              s1=tmp[0].to_i(16)
              s2=tmp[1].to_i(16)
              s=nil
            else
              s=tmp[0].to_i(16)
            end
            f=true if net_chk(s,m,s1,s2,m1,m2)
          end
        end
      end
      return false if not f
      sp-=16
      mp-=16
    end
    return false if sp<mp
    return true if mp==0
    bf=16-mp
    bf=65535<<bf
    m=m1=m2=nil
    s=s1=s2=nil
    if mz>0
      m=0
    else
      mt=ma.shift
      if !mt or mt.empty?
        m=0
      else
        tmp=mt.split("-")
        if tmp[1]
          m1=tmp[0].to_i(16)&bf
          m2=tmp[1].to_i(16)&bf
          m=nil
        else
          m=tmp[0].to_i(16)&bf
        end
      end
    end
    if sz>0
      s=0
      return true if net_chk(s,m,s1,s2,m1,m2)
    else
      st=sa.shift
      if !st or st.empty?
        s=0
        return true if net_chk(s,m,s1,s2,m1,m2)
      else
        for t in st.split(",")
          if t=="*"
            return true
          end
          tmp=t.split("-")
          if tmp[1]
            s1=tmp[0].to_i(16)&bf
            s2=tmp[1].to_i(16)&bf
            s=nil
          else
            s=tmp[0].to_i(16)&bf
          end
          return true if net_chk(s,m,s1,s2,m1,m2)
        end
      end
    end
    return false
  end

  def net_chk(s,m,s1,s2,m1,m2)
    if m and s
      return (s==m)
    end
    if m and !s
      return (m>=s1 and m<=s2)
    end
    if !m and s
      return (s>=m1 and s<=m2)
    end
    return true if m1>=s1 and m1<=s2 or m2>=s1 and m2<=s2
    return true if s1>=m1 and s1<=m2 or s2>=m1 and s2<=m2
    return false
  end
end

class ByMatch
  def initialize(fld=nil)
    @fld=fld
    @mas={}
    return
  end

  def add(s)
    k=s.downcase.split(":")[0]
    @mas[k]=true if k!="-" and k!="?"
    return
  end

  def chk_fld(fields)
    return true if !@fld
    return true if fields.include?(@fld)
    return false
  end

  def match(h,l)
    if !@fld
      l.times do |i|
        next if !h[i] or h[i]=="-" or h[i]=="?"
        s=h[i].downcase.split(":")[0]
        return true if @mas[s]
      end
      return false
    end
    return false if !s=h[@fld]
    s=s.downcase.split(":")[0]
    return (@mas[s]) ? true : false
  end
end

class Line
  attr_reader :b, :la, :ca, :flds

  def initialize(l,b=nil)
    @ca=[]
    if l.class==Line
      @la=[]
      @lflds=[]
      @b=l.b
      return
    end
    @la=[l]
    @lflds=[]
    @b=b
  end

  def add(l,t=nil)
    @la<<l
    if t
      t["__"]=l
      @lflds<<t
    end
  end

  def addfirst(l,t=nil)
    return if @la.include?(l)
    @la.unshift(l)
    if t
      t["__"]=l
      @lflds.unshift(t)
    end
  end

  def addcl(l)
    @ca<<l
  end

  def pr_join(f,v)
    @flds={}
    @flds[f]=v
  end

  def setfld(f,v)
    @flds[f]=(!v or v.empty?) ? ["-"] : v
  end

  def fld(f)
    @flds[f].join("\n")
  end

  def get_all_flds1()
    res={}
    for k,v in @flds
      res[k]=(@flds[k].length==1) ? v[0] : "???"
    end
    for k in @fields
      res[k]=nil if not res.has_key?(k)
    end
    return res
  end

  def get_flds1()
    @fields.length.times do |i|
      next if !@flds[i]
      s=@flds[i][0]
      next if !s or s=="-"
      yield s.downcase.split(":")[0]
    end
  end

  def empty?
    @la.empty?
  end

  def nodata?(fld)
    return true if @flds[0][0]=="-" or @flds[0][0]=="?"
    return true if !@flds[fld]
    return true if @flds[fld][0]=="-" or @flds[fld][0]=="?"
    return false
  end

  def mk_flds()
    @flds={}
    for t in @lflds
      for k,v in t
        @flds[k]=[] if !@flds[k]
        @flds[k]<<v
      end
    end
  end

  def mk_flds_n(fields)
    @fields=fields
    fields.length.times do |i|
      @flds[i]=@flds[fields[i]]
    end
  end

  def mk_la(col)
    @la=[]
    @lflds=[]
    n=0
    while true
      tmp=""
      tab=""
      nok=true
      fok=false
      col.length.times do |i|
        if str=@flds[i][n]
          tmp+=tab if tab
          tmp+=str
          tab="\t"*(col[i]-(str.length/8))
          nok=false
          fok=true
        else
          if fok
            tab+="-"
            tab+="\t"*col[i]
          else
            tab+="\t"*col[i]
          end
        end
      end
      break if nok
      @la<<tmp
      h={}
      for k,v in @flds
        h[k]=v[n]
      end
      @lflds<<h
      n+=1
    end
  end

  def parse(fields,ft,vlan=nil)
    @fields=fields
    @ft=ft
    @flds={}
    @flds["comment"]=[]
    @flds["vlan"]=[vlan] if vlan
    @lflds=[]
    return if fields.empty?
    for l in @la
      @lflds<<{}
      @lflds.last["__"]=l
      tmp=l.split(/\t+/)
      if tmp[0].empty?
        n=tmp.length
        next if n<2
        tc=/\t+/.match(l)[0].length
        i=ft.index(tc)
        next if !i or i<1
        j=1
        while n>1
          f=fields[i]
          @flds[f]=[] if !@flds[f]
          @flds[i]=[] if !@flds[i]
          if tmp[j].start_with?("!")
            str=tmp[j].sub(/^! ?/,"")
            @flds["comment"]<<str
            @lflds.last["comment"]=str
            break
          end
          @flds[f]<<tmp[j]
          @flds[i]<<tmp[j]
          @lflds.last[f]=tmp[j]
          @lflds.last[i]=tmp[j]
          i+=1
          j+=1
          n-=1
        end
        next
      end
      fields.length.times do |i|
        next if !tmp[i] or tmp[i].empty?
        f=fields[i]
        @flds[f]=[] if !@flds[f]
        @flds[i]=[] if !@flds[i]
        if tmp[i].start_with?("!")
          str=tmp[i].sub(/^! ?/,"")
          @flds["comment"]<<str
          @lflds.last["comment"]=str
          break
        end
        s=(i==0) ? tmp[i].sub(/^  /,"") : tmp[i]
        @flds[f]<<s
        @flds[i]<<s
        @lflds.last[f]=s
        @lflds.last[i]=s
      end
    end
  end

  def match(m)
    return self if !m
    if not $sqs
      if m.class==Array
        for l in @la
          for u in m
            return ($im) ? nil : self if u.match(l)
          end
        end
        return ($im) ? self : nil
      end
      if m.class==Regexp
        for l in @la
          return ($im) ? nil : self if m.match(l)
        end
        return ($im) ? self : nil
      end
      if m.class==FldMatch
        for t in @lflds
          return (m.im) ? nil : self if m.match(t)
        end
        return (m.im) ? self : nil
      end
      if m.class==ByMatch
        for t in @lflds
          return ($im) ? nil : self if m.match(t,@fields.length)
        end
        return ($im) ? self : nil
      end
      return nil
    end
    if m.class==Array
      r=Line.new(self)
      for t in @lflds
        f=false
        for u in m
          next if not u.match(t["__"])
          f=true
          break
        end
        r.add(t["__"],t) if f^$im
      end
      return nil if r.empty?
      r.addfirst(@la.first,@lflds.first)
      r.mk_flds
      return r
    end
    if m.class==Regexp
      r=Line.new(self)
      for t in @lflds
        f=(m.match(t["__"])) ? true : false
        r.add(t["__"],t) if f^$im
      end
      return nil if r.empty?
      r.addfirst(@la.first,@lflds.first)
      r.mk_flds
      return r
    end
    if m.class==FldMatch
      r=Line.new(self)
      for t in @lflds
        f=(m.match(t)) ? true : false
        r.add(t["__"],t) if f^m.im
      end
      return nil if r.empty?
      r.addfirst(@la.first,@lflds.first)
      r.mk_flds
      return r
    end
    if m.class==ByMatch
      r=Line.new(self)
      for t in @lflds
        f=(m.match(t,@fields.length)) ? true : false
        r.add(t["__"],t) if f^$im
      end
      return nil if r.empty?
      r.addfirst(@la.first,@lflds.first)
      r.mk_flds
      return r
    end
    return nil
  end
end

class Block
  attr_reader :fields

  def initialize(hdr,fld,sep)
    @lines=[]
    @hdr=hdr
    @fld=fld
    @sep=sep
    @vlan=nil
    if tmp=/^# .* \(([0-9]*)\)/.match(hdr)
      @vlan=tmp[1]
    end
    @fields,@ft=get_fields(fld)
    @joined=false
    @res=nil
  end

  def addline(s)
    if /^\t/.match(s)
      @lines.last.add(s)
      return
    end
    if /^  /.match(s)
      @lines.push(Line.new(s,@lb))
      return
    end
    l=Line.new(s)
    @lines.push(l)
    @lb=l
  end

  def output(out)
    if @joined
      output_j(out)
      return
    end
    return if !@res
    if not $expr.empty?
      res,agr=_exec(@res)
      if $expr.first.show?
        res=_sort(res) if $sort
        output_r(res,out)
        return
      end
      output_e(res,agr,out)
      return
    end
    res=@res
    res=_sort(res) if $sort
    output_r(res,out)
  end

  def block_ok?()
    return false if /^#!/.match(@hdr) and not $wib
    return false if not $catl.empty? and not in_list(@hdr,$catl)
    return false if not $incl.empty? and not in_list(@hdr,$incl)
    return false if in_list(@hdr,$excl)
    return true
  end

  def find(m)
    lines=(@res) ? @res : @lines
    @res=nil
    return false if /^#!/.match(@hdr) and not $wib
    return false if not $catl.empty? and not in_list(@hdr,$catl)
    return false if not $incl.empty? and not in_list(@hdr,$incl)
    return false if in_list(@hdr,$excl)
    if m.class==FldMatch or m.class==ByMatch
      return false if not m.chk_fld(@fields)
      m.prepare if m.class==FldMatch
    end
    res=[]
    ba=[]
    bm=false
    bl=nil
    for l in lines
      l.parse(@fields,@ft,@vlan)
      if $dbm
        if bm and l.b
          res<<l
          next
        end
        if !l.b
          bm=false
          ba.clear
        end
        ba<<l
      end
      bl=l if !l.b
      if t=l.match(m)
        bl=t if !t.b
        if $dbm
          bm=true
          res+=ba
        else
          res<<bl if bl and bl!=t and not res.include?(bl)
          res<<t
        end
      end
    end
    return false if res.empty?
    res=@lines if $dwb
    res=_dups(res) if $dups
    @res=res
    return true
  end

  def chk(h,f)
    @res=nil
    res=[]
    for l in @lines
      l.parse(@fields,@ft)
      next if !l.flds[f] or l.flds[f].length!=1
      k=l.flds[f][0].downcase.split(":")[0]
      next if k=="-" or k=="?"
      nok=(h[k]) ? false : true
      nok^=$im
      if nok
        res<<l.b if l.b and not res.include?(l.b)
        res<<l
      end
    end
    return false if res.empty?
    res=@lines if $dwb
    res=_dups(res) if $dups
    @res=res
    return true
  end

  def join(buf)
    flds=$join.split(",")
    tmp=flds.dup
    jfld=tmp.shift
    jflds=tmp
    if !jidx=@fields.index(jfld)
      return false
    end
    res=[]
    hr={}
    for l in @res
      next if l.nodata?(jfld)
      r=Line.new(l)
      k=l.fld(jfld).downcase
      r.pr_join(jfld,l.flds[jfld])
      for f in jflds
        if @fields.include?(f)
          r.setfld(f,l.flds[f])
        else
          r.setfld(f,nil)
        end
      end
      res<<r
      hr[k]=r
    end
    inbl=igbl=false
    hdr=fld=nil
    fields=ft=nil
    ca=[]
    for s in buf
      if inbl
        if /^--+/.match(s)
          for l in ca
            l.parse(fields,ft)
            k=l.fld(jfld).downcase
            next if !r=hr[k]
            for f in jflds
              if fields.include?(f)
                r.setfld(f,l.flds[f])
              end
            end
          end
          inbl=igbl=false
          hdr=fld=nil
          fields=ft=nil
          ca=[]
          next
        end
        next if igbl
        if /^\t/.match(s)
          ca.last.add(s)
        else
          ca.push(Line.new(s))
        end
        next
      end
      if /^#/.match(s)
        hdr=s
        next
      end
      if /^--+/.match(s)
        fields,ft=get_fields(fld)
        inbl=true
        igbl=(fields.include?(jfld)) ? false : true
        igbl=true if not $inclj.empty? and not in_list(hdr,$inclj)
        igbl=true if in_list(hdr,$exclj)
        next
      end
      if hdr
        fld=s
        next
      end
    end
    @flds=flds
    @fields=flds
    @ft=nil
    @res=res
    @joined=true
    return true
  end

  private

  def output_r(res,out)
    return if res.empty?
    if $istty
      out.print "\e[#{$col}m",@hdr,"\e[m\n"
    else
      out.puts "\n" if $output_cnt>0
      out.puts @hdr
    end
    $output_cnt+=1
    return if $dbh
    if $cnt
      c=res.length
      $aggr.add_cnt(c) if $aggr
      out.print "Count: ",c,"\n"
      return
    end
    out.puts @fld,@sep
    for l in res
      l.la.each { |s| out.puts s }
    end
    out.puts @sep
  end

  def output_e(res,agr,out)
    if res.empty?
      if $istty
        out.print "\e[#{$col}m",@hdr,"\e[m\n"
      else
        out.puts "\n" if $output_cnt>0
        out.puts @hdr
      end
      $output_cnt+=1
      agr.each { |l| out.puts l }
      return
    end
    buf=[]
    fld=[]
    col=[]
    for e in $expr
      next if e.temp?
      next if e.aggr?(1)
      fld<<e.name
      col<<e.name.length/8+1
    end
    lcs=fld.last.length
    for r in res
      r.length.times do |i|
        c=r[i].length/8+1
        col[i]=c if c>col[i]
      end
      z=r.length-1
      lcs=r[z].length if lcs<r[z].length
    end
    mcs=0
    (col.length-1).times { |i| mcs+=col[i]*8 }
    mcs+=lcs
    if mcs>$rs
      $stderr.puts ME+"row length exceeded in expr output"
      exit
    end
    tmp=""
    tab=nil
    fld.length.times do |i|
      tmp+=tab if tab
      tmp+=fld[i]
      tab="\t"*(col[i]-(fld[i].length/8))
    end
    fld=tmp
    for r in res
      tmp=""
      tab=nil
      r.length.times do |i|
        tmp+=tab if tab
        tmp+=r[i]
        tab="\t"*(col[i]-(r[i].length/8))
      end
      buf<<tmp
    end
    sep="-"*mcs
    if $istty
      out.print "\e[#{$col}m",@hdr,"\e[m\n"
    else
      out.puts "\n" if $output_cnt>0
      out.puts @hdr
    end
    $output_cnt+=1
    out.puts fld,sep
    buf.each { |l| out.puts l }
    out.puts sep
    return if agr.empty?
    if $istty
      out.print "\e[#{$col}m",@hdr,"\e[m\n"
    else
      out.puts "\n" if $output_cnt>0
      out.puts @hdr
    end
    $output_cnt+=1
    agr.each { |l| out.puts l }
  end

  def output_j(out)
    return if !@res
    buf=[]
    fld=[]
    col=[]
    for f in @flds
      fld<<f
      col<<f.length/8+1
    end
    lcs=fld.last.length
    for r in @res
      r.mk_flds_n(@fields)
      @fields.length.times do |i|
        for v in r.flds[i]
          c=v.length/8+1
          col[i]=c if c>col[i]
        end
      end
      z=@fields.length-1
      for v in r.flds[z]
        lcs=v.length if lcs<v.length
      end
    end
    mcs=0
    (col.length-1).times { |i| mcs+=col[i]*8 }
    mcs+=lcs
    if mcs>$rs
      $stderr.puts ME+"row length exceeded in join output"
      exit
    end
    tmp=""
    tab=nil
    fld.length.times do |i|
      tmp+=tab if tab
      tmp+=fld[i]
      tab="\t"*(col[i]-(fld[i].length/8))
    end
    fld=tmp
    @res.each { |r| r.mk_la(col) }
    @res=_sort(@res) if $sort
    sep="-"*mcs
    if $istty
      out.print "\e[#{$col}m",@hdr,"\e[m\n"
    else
      out.puts "\n" if $output_cnt>0
      out.puts @hdr
    end
    $output_cnt+=1
    out.puts fld,sep
    for l in @res
      l.la.each { |s| out.puts s }
    end
    out.puts sep
  end

  def _dups(ra)
    res=[]
    for l in ra
      s=l.flds[0][0].downcase
      next if s.empty? or s=="-" or s=="?"
      next if !$db[s]
      res<<l.b if l.b and not res.include?(l.b)
      res<<l
    end
    return res
  end

  def _exec(ra)
    res=[]
    agr=[]
    av={}
    ar={}
    vars={}
    saggr=false
    for e in $expr
      av[e]=nil
      ar[e]=[]
      saggr=true if e.aggr?(2)
    end
    for l in ra
      tmp=[]
      fld=l.get_all_flds1
      bl=(l.b) ? true : false
      if saggr and not bl and not vars.empty?
        for e in $expr
          av[e]=e.exec({},ar[e],vars,bl) if e.aggr?(5)
        end
      end
      i=0
      for e in $expr
        if e.show?
          next if e.exec(fld,ar[e],vars)!=1
          res<<l.b if bl and not res.include?(l.b)
          res<<l
          next
        end
        if e.aggr?(1)
          next if saggr and bl
          av[e]=e.exec(fld,ar[e],vars,bl) if not e.aggr?(4)
          next
        end
        v=e.exec(fld,ar[e],vars,bl)._str(3)
        next if e.temp?
        if saggr and bl
          res.last[i]=v if e.aggr?(2,4)
        else
          tmp<<v
        end
        i+=1
      end
      res<<tmp if not tmp.empty?
    end
    if saggr and bl
      for e in $expr
        av[e]=e.exec({},ar[e],vars,bl) if e.aggr?(5)
      end
    end
    for e in $expr
      next if not e.aggr?(1)
      agr<<e.name+": "+av[e]._str(3)
    end
    $aggr.add_agr(av) if $aggr
    return res,agr
  end

  def _sort(la)
    sa=[]
    for s in $sort.split(",")
      sa.push(s) if @fields.include?(s)
    end
    return la if sa.empty?
    ra=[]
    for l in la
      if l.b
        ra.last.addcl(l)
      else
        ra<<l
      end
    end
    ra.sort! do |x,y|
      r=nil
      for s in sa
        r=_comp(x.fld(s),y.fld(s))
        next if r==0
        break
      end
      r
    end
    for l in ra
      l.ca.sort! do |x,y|
        r=nil
        for s in sa
          r=_comp(x.fld(s),y.fld(s))
          next if r==0
          break
        end
        r
      end
    end
    res=[]
    for l in ra
      res<<l
      res.concat(l.ca)
    end
    return res
  end

  def _comp(x,y)
    if $sortmode==1
      return (x<=>y)
    end
    return 0 if x.empty? and y.empty?
    return -1 if x.empty?
    return 1 if y.empty?
    xa=x.scan(/([0-9]+)|([^0-9]+)/)
    ya=y.scan(/([0-9]+)|([^0-9]+)/)
    r=nil
    xi=xa.shift
    yi=ya.shift
    while xi and yi
      if xi[0] and yi[0]
        xv=xi[0].to_i
        yv=yi[0].to_i
        if(xv==yv)
          r=0
          xi=xa.shift
          yi=ya.shift
          next
        end
        r=(xv>yv) ? 1 : -1
        break
      end
      if xi[1] and yi[1]
        xv=xi[1]
        yv=yi[1]
        if(xv==yv)
          r=0
          xi=xa.shift
          yi=ya.shift
          next
        end
        r=(xv<=>yv)
        break
      end
      break
    end
    return (x<=>y) if !r
    if r==0
      r=-1 if !xi and yi
      r=1 if xi and !yi
    end
    return r
  end
end

class Block
  def output_csv(out)
    return if !@res
    str=""
    d=nil
    for s in @fields
      str+=d if d
      str+=s
      d=","
    end
    out.puts str
    c=@fields.length
    for l in @res
      l.parse(@fields,@ft)
      str=""
      d=nil
      c.times do |i|
        str+=d if d
        s=(l.flds[i]) ? l.fld(i) : ""
        s.gsub!(/"/,'""')
        s='"'+s+'"' if /[ ,"\n]/.match(s)
        str+=s
        d=","
      end
      out.puts str
    end
  end
end

class Aggr
  def output(out)
    return if $cnt and @cnt==0
    return if @agr and @agr.empty?
    if $istty
      out.print "\e[#{$col}m+ Sum (Aggregated)\e[m\n"
    else
      out.puts "\n" if $output_cnt>0
      out.puts "+ Sum (Aggregated)"
    end
    $output_cnt+=1
    if $cnt
      out.print "Count: ",@cnt,"\n"
      return
    end
    for e in $expr
      next if not e.aggr?(1)
      out.print e.name,": ",@agr[e]._str(3),"\n"
    end
  end
end

class FldMatch
  def initialize(str)
    @mas=nil
    @fld=nil
    @im=false
    @em=false
    @type=nil
    tmp=/([^!=]*)(!?==?)(.*)/.match(str.downcase)
    @sfld=tmp[1]
    @str=tmp[3]
    @cmp=tmp[2]
  end

  def prepare
    if $lst_fields.include?(@fld)
      @type="lst"
      @mas=rexpand(@str)
      if @cmp=="==" or @cmp=="!=="
        @em=true
      end
      @im=true if @cmp.start_with?("!")
      return
    end
    if $ip_fields.include?(@fld)
      @type="ip"
      @mas=@str
      @im=true if @cmp=="!="
      return
    end
    if $id_fields.include?(@fld)
      @type="id"
      @mas=@str
      @im=true if @cmp=="!="
      return
    end
    if $ifn_fields.include?(@fld)
      @type="ifn"
      if @cmp=="==" or @cmp=="!=="
        @mas=@str
        @em=true
      else
        @mas=Regexp.new(regexfix(@str),true)
      end
      @im=true if @cmp.start_with?("!")
      return
    end
    @type="str"
    if @cmp=="==" or @cmp=="!=="
      @mas=@str
      @em=true
    else
      @mas=Regexp.new(regexfix(@str),true)
    end
    @im=true if @cmp.start_with?("!")
    return
  end
end

def p_output(blocks)
  return if blocks.empty?
  if $csv
    if blocks.length>1
      $stderr.puts ME+"output from multiple blocks is not supported"
      exit
    end
  end
  if $istty
    out=IO.popen("less -fqRFX","w")
  else
    out=$stdout
  end
  begin
    if $csv
      blocks.first.output_csv(out)
    elsif $dbh and $cnt
      out.print "Count: ",blocks.length,"\n"
    else
      blocks.each { |b| b.output(out) }
      $aggr.output(out) if $aggr
    end
  rescue SystemExit
    out.close if out!=$stdout
    exit
  rescue Errno::EPIPE
  end
  out.close if out!=$stdout
end

def i_fld()
  buf=[]
  for l in $stdin
    l.chop!
    next if l.empty?
    next if /^! /.match(l)
    next if /^<[^>]*>$/.match(l)
    buf.push(l)
  end
  tmp=$fld.split(":")
  f1=tmp[0]
  f2=(tmp[1]) ? tmp[1] : tmp[0]
  m=ByMatch.new(f2)
  inbl=igbl=false
  hdr=fld=nil
  fields=ft=nil
  ca=[]
  for s in buf
    if inbl
      if /^--+/.match(s)
        for l in ca
          l.parse(fields,ft)
          next if !l.flds[f1]
          l.flds[f1].each { |v| m.add(v) }
        end
        inbl=igbl=false
        hdr=fld=nil
        fields=ft=nil
        ca=[]
        next
      end
      next if igbl
      if /^\t/.match(s)
        ca.last.add(s)
      else
        ca.push(Line.new(s))
      end
      next
    end
    if /^#/.match(s)
      hdr=s
      next
    end
    if /^--+/.match(s)
      inbl=true
      fields,ft=get_fields(fld)
      igbl=true if not fields.include?(f1)
      next
    end
    fld=s if hdr
  end
  return m
end

def i_fldm()
  buf=[]
  for l in $stdin
    l.chop!
    next if l.empty?
    next if /^! /.match(l)
    next if /^<[^>]*>$/.match(l)
    buf.push(l)
  end
  m=ByMatch.new
  inbl=igbl=false
  hdr=fld=nil
  fields=ft=nil
  ca=[]
  for s in buf
    if inbl
      if /^--+/.match(s)
        for l in ca
          l.parse(fields,ft)
          next if !l.flds[$fldm]
          for t in l.flds[$fldm]
            t.split(" ").each { |v| m.add(v) }
          end
        end
        inbl=igbl=false
        hdr=fld=nil
        fields=ft=nil
        ca=[]
        next
      end
      next if igbl
      if /^\t/.match(s)
        ca.last.add(s)
      else
        ca.push(Line.new(s))
      end
      next
    end
    if /^#/.match(s)
      hdr=s
      next
    end
    if /^--+/.match(s)
      inbl=true
      fields,ft=get_fields(fld)
      igbl=true if not fields.include?($fldm)
      next
    end
    fld=s if hdr
  end
  return m
end

def i_parse(buf)
  blocks=[]
  inbl=false
  hdr=fld=nil
  bl=nil
  tmp={}
  for s in buf
    if inbl
      if /^--+/.match(s)
        inbl=false
        hdr=fld=nil
        blocks.push(bl) if bl
        bl=nil
        next
      end
      next if !bl
      bl.addline(s)
      if $dups
        s=s.sub(/^  /,"").downcase.split(/\t+/)[0]
        next if s.empty? or s=="-" or s=="?"
        if tmp[s]
          $db[s]=true
        else
          tmp[s]=true
        end
      end
      next
    end
    if /^#/.match(s)
      hdr=s
      next
    end
    if /^--+/.match(s)
      inbl=true
      bl=Block.new(hdr,fld,s) if hdr and fld
      next
    end
    fld=s if hdr
  end
  return blocks
end

def load_file(buf,fn)
  dir=File.dirname(fn)
  begin
    f=File.open(fn,"r")
  rescue
    $stderr.puts ME+"unable to open file"
    exit
  end

  for l in f
    l.chop!
    next if l.empty?
    next if /^! /.match(l)
    buf.push(l)
  end
  f.close
end

def usage
  me=ME.delete(":")
  puts "Usage: "+me+"[-B <pattern>] [-S <pattern>] <pattern> [<file>] ..."
  puts "  -i              - include ignored blocks"
  puts "  -n              - check for duplicate items"
  puts "  -c              - count matches"
  puts "  -a              - show aggregated count and expr"
  puts "  -v              - invert match"
  puts "  -w              - display whole block on match"
  puts "  -b              - display block on match"
  puts "  -q              - display only matching lines"
  puts "  -h              - display only block headers"
  puts "  -C <pattern>    - display blocks matching RegEx <pattern>"
  puts "  -B <pattern>    - include blocks matching RegEx <pattern>"
  puts "  -S <pattern>    - skip blocks matching RegEx <pattern>"
  puts "  -Bj <pattern>   - include blocks matching RegEx <pattern> (in join)"
  puts "  -Sj <pattern>   - skip blocks matching RegEx <pattern> (in join)"
  puts "  -m <pattern>    - match <pattern> both in blocks and lines"
  puts "  -e <expr>       - do simple math expr on every record"
  puts "  -k <field>      - sort result by field"
  puts "  -K <field>      - sort result by field (natural order)"
  puts "  -by <file>      - select from file/stdin"
  puts "  -fld <field>    - simple crossjoin by <field>"
  puts "  -chk <field>    - check if entry exist (crossjoin by <field>)"
  puts "  -j <field>,...  - join multiple files by <field>"
  exit
end

def parse_opts(opts)
  i=0
  while i<opts.length
    if opts[i]=="--"
      i+=1
      break
    end
    break if opts[i]=="-"
    break if not opts[i].start_with?("-")
    usage if opts[i]=="--help"
    if opts[i]=="--version"
      me=ME.delete(":")
      puts me+VER+" by Borg."
      exit
    end

    if opts[i]=="-l"
      i+=1
      next
    end
    if ARGV[i]=="-long"
      i+=1
      $rs=254
      next
    end
    if opts[i]=="-csv"
      i+=1
      $csv=true
      next
    end
    if opts[i]=="-i"
      i+=1
      $wib=true
      next
    end
    if opts[i]=="-n"
      i+=1
      $dups=true
      next
    end
    if opts[i]=="-c"
      i+=1
      $cnt=true
      next
    end
    if opts[i]=="-a"
      i+=1
      $aggr=Aggr.new
      next
    end
    if opts[i]=="-v"
      i+=1
      $im=true
      next
    end
    if opts[i]=="-w"
      i+=1
      $dwb=true
      next
    end
    if opts[i]=="-q"
      i+=1
      $sqs=true
      next
    end
    if opts[i]=="-h"
      i+=1
      $dbh=true
      next
    end
    if opts[i]=="-b"
      i+=1
      $dbm=true
      next
    end
    if opts[i]=="-m"
      i+=1
      if !opts[i]
        $stderr.puts ME+"Need argument for '-m' option"
        exit
      end
      $incl.push(Regexp.new(regexfix2(opts[i]),true))
      $ma.push(Regexp.new(regexfix(opts[i]),true))
      i+=1
      next
    end
    if opts[i]=="-by"
      i+=1
      if !opts[i]
        $stderr.puts ME+"Need argument for '-by' option"
        exit
      end
      $by=opts[i]
      i+=1
      next
    end
    if opts[i]=="-fld"
      i+=1
      if !opts[i]
        $stderr.puts ME+"Need argument for '-fld' option"
        exit
      end
      $fld=opts[i].downcase
      i+=1
      next
    end
    if opts[i]=="-fldm"
      i+=1
      if !opts[i]
        $stderr.puts ME+"Need argument for '-fldm' option"
        exit
      end
      $fldm=opts[i].downcase
      i+=1
      next
    end
    if opts[i]=="-chk"
      i+=1
      if !opts[i]
        $stderr.puts ME+"Need argument for '-chk' option"
        exit
      end
      $chk=opts[i].downcase
      i+=1
      next
    end
    if opts[i]=="-j"
      i+=1
      if !opts[i]
        $stderr.puts ME+"Need argument for '-j' option"
        exit
      end
      $join=opts[i].downcase
      i+=1
      next
    end
    if ARGV[i]=="-e"
      i+=1
      if !ARGV[i]
        $stderr.puts ME+"Need argument for '-e' option"
        exit
      end
      $expr+=Expr.parse(ARGV[i])
      i+=1
      next
    end
    if opts[i]=="-C"
      i+=1
      if !opts[i]
        $stderr.puts ME+"Need argument for '-C' option"
        exit
      end
      $wib=true if opts[i].start_with?("!")
      str=regexfix2(opts[i])
      $catl.push(Regexp.new(str,true))
      i+=1
      next
    end
    if opts[i]=="-k"
      i+=1
      if !opts[i]
        $stderr.puts ME+"Need argument for '-k' option"
        exit
      end
      $sort=opts[i]
      $sortmode=1
      i+=1
      next
    end
    if opts[i]=="-K"
      i+=1
      if !opts[i]
        $stderr.puts ME+"Need argument for '-K' option"
        exit
      end
      $sort=opts[i]
      $sortmode=2
      i+=1
      next
    end
    if opts[i]=="-B"
      i+=1
      if !opts[i]
        $stderr.puts ME+"Need argument for '-B' option"
        exit
      end
      $wib=true if opts[i].start_with?("!")
      str=regexfix2(opts[i])
      $incl.push(Regexp.new(str,true))
      i+=1
      next
    end
    if opts[i]=="-S"
      i+=1
      if !opts[i]
        $stderr.puts ME+"Need argument for '-S' option"
        exit
      end
      $wib=true if opts[i].start_with?("!")
      str=regexfix2(opts[i])
      $excl.push(Regexp.new(str,true))
      i+=1
      next
    end
    if opts[i]=="-Bj"
      i+=1
      if !opts[i]
        $stderr.puts ME+"Need argument for '-Bj' option"
        exit
      end
      str=regexfix2(opts[i])
      $inclj.push(Regexp.new(str,true))
      i+=1
      next
    end
    if opts[i]=="-Sj"
      i+=1
      if !opts[i]
        $stderr.puts ME+"Need argument for '-Sj' option"
        exit
      end
      str=regexfix2(opts[i])
      $exclj.push(Regexp.new(str,true))
      i+=1
      next
    end
    $stderr.puts ME+"Unknown option: "+opts[i]
    exit
  end
  return i
end

$istty=$stdout.isatty
if ENV['BLGREP_OPT']
  parse_opts(ENV['BLGREP_OPT'].split(" "))
end
i=parse_opts(ARGV)

if $csv and $cnt
  $stderr.puts ME+"'-csv' and '-c' options cannot be mixed"
  exit
end
if $csv and $chk
  $stderr.puts ME+"'-csv' and '-chk' options cannot be mixed"
  exit
end
if $csv and $fld
  $stderr.puts ME+"'-csv' and '-fld' options cannot be mixed"
  exit
end
if $csv and $fldm
  $stderr.puts ME+"'-csv' and '-fldm' options cannot be mixed"
  exit
end
if $csv and $by
  $stderr.puts ME+"'-csv' and '-by' options cannot be mixed"
  exit
end
if $csv and $join
  $stderr.puts ME+"'-csv' and '-j' options cannot be mixed"
  exit
end
if $cnt and $join
  $stderr.puts ME+"'-c' and '-j' options cannot be mixed"
  exit
end
if $by and $fld
  $stderr.puts ME+"'-by' and '-fld' options cannot be mixed"
  exit
end
if $by and $fldm
  $stderr.puts ME+"'-by' and '-fldm' options cannot be mixed"
  exit
end
if $join and $chk
  $stderr.puts ME+"'-j' and '-chk' options cannot be mixed"
  exit
end
if $join and $by
  $stderr.puts ME+"'-j' and '-by' options cannot be mixed"
  exit
end
if $join and $fld
  $stderr.puts ME+"'-j' and '-fld' options cannot be mixed"
  exit
end
if $join and $fldm
  $stderr.puts ME+"'-j' and '-fldm' options cannot be mixed"
  exit
end
if not $ma.empty? and $join
  $stderr.puts ME+"'-m' and '-j' options cannot be mixed"
  exit
end
if not $ma.empty? and $chk
  $stderr.puts ME+"'-m' and '-chk' options cannot be mixed"
  exit
end
if not $ma.empty? and $by
  $stderr.puts ME+"'-m' and '-by' options cannot be mixed"
  exit
end
if not $ma.empty? and $fld
  $stderr.puts ME+"'-m' and '-fld' options cannot be mixed"
  exit
end
if not $ma.empty? and $fldm
  $stderr.puts ME+"'-m' and '-fldm' options cannot be mixed"
  exit
end
if $fld and $fldm
  $stderr.puts ME+"'-fld' and '-fldm' options cannot be mixed"
  exit
end

if i>=ARGV.length and $catl.empty? and $ma.empty?
  usage
end

if $cnt and not $expr.empty?
  $stderr.puts ME+"'-c' and '-e' options cannot be mixed"
  exit
end
ec=sc=ac=0
for e in $expr
  ec+=1
  sc+=1 if e.show?
  ac+=1 if e.aggr>0
end
if sc>0 and (sc!=ec or ac>0)
  $stderr.puts ME+"expr: show function cannot be mixed"
  exit
end
$aggr=nil if not $cnt and ac<1
$aggr.init if $aggr and ac>0

if $by
  buf=[]
  tmp=$by.split(":")
  fn=tmp[0]
  fld=(tmp[1]) ? tmp[1] : nil
  if fn=="-"
    for l in $stdin
      l.chop!
      next if l.empty?
      buf.push(l)
    end
  else
    begin
      f=File.open(fn,"r")
    rescue
      $stderr.puts ME+"unable to open file"
      exit
    end
    for l in f
      l.chop!
      next if l.empty?
      buf.push(l)
    end
    f.close
  end
  m=ByMatch.new(fld)
  buf.each { |s| m.add(s) }
  buf=[]
  while i<ARGV.length
    fn=ARGV[i]
    i+=1
    load_file(buf,fn)
  end
  if buf.empty?
    $stderr.puts ME+"no files specified"
    exit
  end
  blocks=i_parse(buf)
  res=[]
  for b in blocks
    res<<b if b.find(m)
  end
  p_output(res)
  exit
end

if $chk
  buf=[]
  for l in $stdin
    l.chop!
    next if l.empty?
    next if /^! /.match(l)
    next if /^<[^>]*>$/.match(l)
    buf.push(l)
  end
  inbuf=[]
  while i<ARGV.length
    fn=ARGV[i]
    i+=1
    load_file(inbuf,fn)
  end
  if inbuf.empty?
    $stderr.puts ME+"no files specified"
    exit
  end
  tmp=$chk.split(":")
  f1=tmp[0]
  f2=(tmp[1]) ? tmp[1] : tmp[0]
  inbl=igbl=false
  hdr=fld=nil
  fields,ft=nil
  ca=[]
  ht={}
  for s in inbuf
    if inbl
      if /^--+/.match(s)
        for l in ca
          l.parse(fields,ft)
          next if !l.flds[f2] or l.flds[f2].length!=1
          k=l.flds[f2][0].downcase.split(":")[0]
          next if k=="-" or k=="?"
          ht[k]=true
        end
        inbl=igbl=false
        hdr=fld=nil
        fields,ft=nil
        ca=[]
        next
      end
      next if igbl
      if /^\t/.match(s)
        ca.last.add(s)
      else
        ca.push(Line.new(s))
      end
      next
    end
    if /^#/.match(s)
      hdr=s
      next
    end
    if /^--+/.match(s)
      inbl=true
      fields,ft=get_fields(fld)
      igbl=true if /^#!/.match(hdr) and not $wib
      igbl=true if not $catl.empty? and not in_list(hdr,$catl)
      igbl=true if not $incl.empty? and not in_list(hdr,$incl)
      igbl=true if in_list(hdr,$excl)
      next
    end
    fld=s if hdr
  end
  blocks=[]
  inbl=false
  hdr=fld=nil
  fields=ft=nil
  bl=nil
  for s in buf
    if inbl
      if /^--+/.match(s)
        inbl=false
        hdr=fld=nil
        fields=ft=nil
        blocks.push(bl) if bl
        bl=nil
        next
      end
      bl.addline(s) if bl
      next
    end
    if /^#/.match(s)
      hdr=s
      next
    end
    if /^--+/.match(s)
      inbl=true
      fields,ft=get_fields(fld)
      next if not fields.include?(f1)
      bl=Block.new(hdr,fld,s) if hdr and fld
      next
    end
    fld=s if hdr
  end
  res=[]
  for b in blocks
    res<<b if b.chk(ht,f1)
  end
  p_output(res)
  exit
end

if $join
  if $catl.empty?
    if ARGV[i]=="-"
      m=nil
    elsif ARGV[i].include?("=")
      m=FldMatch.new(ARGV[i])
    else
      m=Regexp.new(regexfix(ARGV[i]),true)
    end
    i+=1
  else
    m=nil
  end
  if i>=ARGV.length
    $stderr.puts ME+"no files specified for join"
    exit
  end
  buf=[]
  fn=ARGV[i]
  i+=1
  if fn=="-"
    for l in $stdin
      l.chop!
      next if l.empty?
      next if /^! /.match(l)
      next if /^<[^>]*>$/.match(l)
      buf.push(l)
    end
  else
    load_file(buf,fn)
  end
  blocks=i_parse(buf)
  res=[]
  for b in blocks
    res<<b if b.find(m)
  end
  blocks=res
  buf=[]
  while i<ARGV.length
    fn=ARGV[i]
    i+=1
    load_file(buf,fn)
  end
  res=[]
  for b in blocks
    res<<b if b.join(buf)
  end
  p_output(res)
  exit
end

ma=[]
if $fld or $fldm
  m=i_fld if $fld
  m=i_fldm if $fldm
  ma<<m
end

if $catl.empty? and $ma.empty?
  if ARGV[i]=="-"
    m=nil
    i+=1
  elsif ARGV[i].include?("=")
    m=FldMatch.new(ARGV[i])
    i+=1
  elsif $fld or $fldm
    m=nil
  else
    m=Regexp.new(regexfix(ARGV[i]),true)
    i+=1
  end
  ma<<m if m
end

buf=[]
if i>=ARGV.length
  $wib=!$wib
  for l in $stdin
    l.chop!
    next if l.empty?
    next if /^! /.match(l)
    next if /^<[^>]*>$/.match(l)
    buf.push(l)
  end
else
  while i<ARGV.length
    fn=ARGV[i]
    i+=1
    load_file(buf,fn)
  end
end
blocks=i_parse(buf)

if not $ma.empty?
  res=[]
  for b in blocks
    res<<b if b.find($ma)
  end
  p_output(res)
  exit
end

ma<<nil if ma.empty?
for m in ma
  res=[]
  for b in blocks
    res<<b if b.find(m)
  end
  blocks=res
end
p_output(res)