# Copyright (C) 2003-2005 Kouichirou Eto, All rights reserved.

$LOAD_PATH << "../../lib" unless $LOAD_PATH.include?("../../lib")

module Qwik
  class TextTokenizer # line oriented tokenizer
    TERMINAL_REGEXP = { # terminate regexp
      :plugin=>/\A\}\}\z/,
      :pre=>/\A\}\}\}\z/,
      :html=>/\A\<\/html\>\z/,
    }

    def tokenize(str, br_mode=false)
      term = TERMINAL_REGEXP
      page_ar = []
      in_tag = {}

      str.each {|line|
	line.chomp!

	# at the first, branch if this line is in tag.
	last_token = page_ar.last
	if last_token
	  last_tag = last_token[0] 
	  if in_tag[last_tag]
	    if term[last_tag] =~ line
	      in_tag[last_tag] = nil
	    else
	      last_token[-1] += line.chomp+"\n"
	    end
	    next
	  end
	end

	line.chomp!

	# preprocess
	#line.gsub!(/&my-([0-9]+);/) {|m| "{{my_char(#{$1})}}" }

	first_character = line[0] # first character
	rest = line[1, line.length-1] # rest of the line

	case first_character
	when nil, ?#	# line is "", or comment.
	  page_ar << [:empty]	# empty line

	when ?*		# h
	  if /\A(\*{1,5})(.+)\z/ =~ line
	    h = $1
	    s = $2.to_s		# Since $2 maybe nil, convert it to str.
	    s = s.strip
	    if 0 < s.length
	      page_ar << [("h#{h.size+1}").intern, s]
	    else	# "* "
	      text(page_ar, line, br_mode)
	    end
	  else		# "*"
	    text(page_ar, line, br_mode)
	  end

	when ?=		# hr
	  if /\A====+\z/ =~ line
	    page_ar << [:hr]
	  else
	    text(page_ar, line, br_mode)
	  end

	when ?-		# ul
	  if line == "-- " || /\A----+\z/ =~ line	# special hr mode
	    page_ar << [:hr]
	  elsif /\A(\-{1,3})(.+)\z/ =~ line
	    page_ar << [:ul, $1.size, $2.to_s.strip]
	  else
	    text(page_ar, line, br_mode)
	  end

	when ?+		# ol
	  if /\A(\+{1,3})(.+)\z/ =~ line
	    page_ar << [:ol, $1.size, $2.to_s.strip]
	  else
	    text(page_ar, line, br_mode)
	  end

	when ?>		# blockquote
	  page_ar << [:blockquote, rest.strip]

	when ?:		# dl
	  dt, dd = rest.split(":", 2)
	  if dt && dt.include?("(")
	    if /\A((?:\(.*?\)|.)*):(.*)\z/ =~ rest	# bad hack
	      dt, dd = $1, $2
	    end
	  end
	  ar = [:dl]
	  ar << ((dt && dt != "") ? dt.to_s.strip : nil)
	  ar << ((dd && dd != "") ? dd.to_s.strip : nil)
	  page_ar << ar

	when ?\s, ?\t		# pre
	  page_ar << [:pre, rest]

	when ?, , ?|		# table
	  re = Regexp.new(Regexp.quote(first_character.chr), nil, "s")
	  ar = [:table] + rest.split(re).map {|a| a.to_s.strip }
	  page_ar << ar

	when ?{		# plugin or super pre
	  if /\A\{\{([^\(\)\{\}]+?)(?:\(([^\(\)\{\}]*?)\))?\}\}\z/ =~ line
	    page_ar << [:plugin, $1.to_s, $2.to_s]
	  elsif /\A\{\{([^\(\)\{\}]+?)(?:\(([^\(\)\{\}]*?)\))?\z/ =~ line
	    in_tag[:plugin] = true
	    page_ar << [:plugin, $1.to_s, $2.to_s, ""]
	  elsif /\A\{\{\{/ =~ line
	    in_tag[:pre] = true
	    page_ar << [:pre, ""]
	  else
	    text(page_ar, line, br_mode)
	  end

	when ?<		# <html>
	  if line == "<html>"
	    page_ar << [:html, ""]
	    in_tag[:html] = true
	  else
	    text(page_ar, line, br_mode)
	  end

	else		# normal text
	  text(page_ar, line, br_mode)
	end
      }
      page_ar
    end

    private

    def text(page_ar, line, br_mode)
      if /~\z/s =~ line
	line = line.sub(/~\z/s, "{{br}}") # //s means shift_jis
      elsif br_mode
	line = line+"{{br}}"
      end
      page_ar << [:text, line]
    end
  end

  module InlinePatternModule
    URL = '(?:http|https|ftp|mailto|file):[a-zA-Z0-9;/?:@&=+$,\-_.!~*\'()#%]+'
  end

  class InlineTokenizer
    include InlinePatternModule

    SPECIAL = '^\[\]\'=\{\}'

    def tokenize(str) # split a line into tokens
      line_ar = []

      while 0 < str.length
	first_character = str[0]
	rest = str[1, str.length-1]

	case first_character
	when ?'		# ''t'' or '''t'''
	  if /\A'''(?!:')/ =~ str
	    line_ar << :"'''"
	    str = $'
	  elsif /\A''/ =~ str
	    line_ar << :"''"
	    str = $'
	  else
	    line_ar << first_character.chr
	    str = rest
	  end

	when ?=		# ==t==
	  if /\A==/ =~ str
	    line_ar << :"=="
	    str = $'
	  else
	    line_ar << first_character.chr
	    str = rest
	  end

	when ?[		# [[t]]
	  if /\A\[\[(.+?)\]\]/ =~ str
	    line_ar << [:ref, $1]
	    str = $'
	  else
	    line_ar << first_character.chr
	    str = rest
	  end

	when ?{		# {{t}}
	  if /\A\{\{([^\(\)]+?)(?:\((.*?)\))?\s*\}\}/ =~ str # {{t(a)}}
	    ar = [:plugin, $1]
	    ar << $2 if $2
	    line_ar << ar
	    str = $'
	  else
	    line_ar << first_character.chr
	    str = rest
	  end

	else
	  if /\A#{URL}/s =~ str
	    href = $&
	    line_ar << [:url, href]
	    str  = $'

	  elsif /\A[^#{SPECIAL}]+/s =~ str
	    m = $&
	    after = $'

	    if /([^a-zA-Z\d]+)(#{URL})/ =~ m
	      s = $` + $1
	      line_ar << s
	      str = $2 + $' + after
	    else
	      line_ar << m
	      str = after
	    end

	  else
	    if /\A(.+?)([^#{SPECIAL}])/ =~ str
	      line_ar << $1
	      str = $2 + $'
	    else
	      line_ar << str
	      str = ""
	    end
	  end

	end
      end

      line_ar
    end

  end
end

if $0 == __FILE__
  require "test/unit"
  $debug = true
end

if $debug
  class TC_TextTokenizer < Test::Unit::TestCase
    def assert(e, str)
      assert_equal(e, Qwik::TextTokenizer.new.tokenize(str))
    end

    def test_all
      assert([], "")
      assert([[:empty]], "#t")
      assert([[:hr]], "====")
      assert([[:text, "==="]], "===")
      assert([[:text, "*"]], "*")
      assert([[:h2, "t"]], "*t")
      assert([[:h2, "t"]], "* t")
      assert([[:text, "* "]], "* ")
      assert([[:text, "*"]], "*")
      assert([[:h3, "t"]], "**t")
      assert([[:h4, "t"]], "***t")
      assert([[:h5, "t"]], "****t")
      assert([[:h6, "t"]], "*****t")
      assert([[:text, "-"]], "-")
      assert([[:ul, 1, "t"]], "-t")
      assert([[:ul, 1, "t"]], "- t")
      assert([[:ul, 2, "t"]], "--t")
      assert([[:ul, 3, "t"]], "---t")
      assert([[:ul, 3, "-t"]], "----t") # uum...
      assert([[:ul, 1, "-"]], "--")
      assert([[:hr]], "-- ")
      assert([[:hr]], "----")
      assert([[:text, "+"]], "+")
      assert([[:ol, 1, "t"]], "+t")
      assert([[:blockquote, "t"]], ">t")
      assert([[:blockquote, "t"]], "> t")

      # test dl
      assert([[:dl, "dt", "dd"]], ":dt:dd")
      assert([[:dl, "dt", nil]], ":dt")
      assert([[:dl, nil, nil]], ":")
      assert([[:dl, nil, "dd"]], "::dd")
      assert([[:dl, nil, nil]], "::")
      assert([[:dl, "dt", "dd"], [:dl, "dt2", "dd2"]], ":dt:dd\n:dt2:dd2")

      assert([[:pre, "t"]], " t")
      assert([[:pre, "t"]], "\tt")

      # test_table
      assert([[:table, "t"]], ",t")
      assert([[:table, "t"]], ", t")
      assert([[:table, "s", "t"]], ",s,t")
      assert([[:table, "", "t"]], ",,t")
      assert([[:table, "t"]], "|t")
      assert([[:table, "$t"]], ",$t")
      assert([[:table, "$1"]], ",$1")

      assert([[:text, "{t}"]], "{t}")
      assert([[:plugin, "t", ""]], "{{t}}")
      assert([[:plugin, "t", "a"]], "{{t(a)}}")
      assert([[:plugin, "t", "", ""]], "{{t")
      assert([[:plugin, "t", "a", ""]], "{{t(a)")
      assert([[:plugin, "t", "", "s\n"]], "{{t\ns\n}}")
      assert([[:pre, "s\n"]], "{{{\ns\n}}}")
      assert([[:pre, "s\nt\n"]], "{{{\ns\nt\n}}}")
      assert([[:empty]], "\n")
      assert([[:text, "t"]], "t")
      assert([[:text, "s"], [:text, "t"]], "s\nt")
      assert([[:text, "s"], [:empty], [:text, "t"]], "s\n\nt")

      # test_html
      assert([[:text, "<t"]], "<t")
      assert([[:html, ""]], "<html>")
      assert([[:html, "t\n"]], "<html>\nt\n</html>")
      assert([[:html, "<p>\nt\n</p>\n"]], "<html>\n<p>\nt\n</p>\n</html>")

      # test_sjis_bug
      assert([[:table, "•|", "•|"]], ",•|,•|")
      assert([[:table, "s", "t"]], "|s|t")
      assert([[:table, "•|", "•|"]], "|•||•|")

      # test_multiline
      assert([[:text, "s"], [:text, "t"]], "s\nt")
      assert([[:text, "s{{br}}"], [:text, "t{{br}}"]], "s~\nt~")
      assert([[:text, "s"], [:empty], [:text, "t"]], "s\n\nt")
      assert([[:text, "s{{br}}"], [:text, "{{br}}"], [:text, "t{{br}}"]],
	     "s~\n~\nt~")

      # test_ul
      assert([[:ul, 1, "t"], [:ul, 1, "t"]], "-t\n-t")
      assert([[:ul, 1, "t"], [:empty], [:ul, 1, "t"]], "-t\n\n-t")
      assert([[:empty], [:ul, 1, "t"], [:ul, 1, "t"]], "\n-t\n-t")

      # test_multiline_pre
      assert([[:pre, "t"]], " t")
      assert([[:pre, "t1"], [:pre, "t2"]], " t1\n t2")
      assert([[:pre, "t1"], [:empty], [:pre, "t2"]], " t1\n\n t2")

      # test_ref
      # do not parse inline for this moment.
      assert([[:text, "[[t]]"]], "[[t]]")

      # test_super_pre
      assert([[:pre, "p\n"], [:text, "t"]], "{{{\np\n}}}\nt\n")

      # test_in_bracket # bad hack
      assert([[:dl, "dt", "dd"]], ":dt:dd")
      assert([[:dl, "dt(h:m)dt2", "dd"]], ":dt(h:m)dt2:dd")
      assert([[:dl, "u", "h://e.com/"]], ":u:h://e.com/")
    end
  end

  class TC_TextTokenizerWithBR < Test::Unit::TestCase
    def assert(e, str)
      assert_equal(e, Qwik::TextTokenizer.new.tokenize(str, true))
    end

    def test_all
      assert([[:text, "==={{br}}"]], "===")
      assert([[:text, "*{{br}}"]], "*")
      assert([[:text, "* {{br}}"]], "* ")
      assert([[:text, "*{{br}}"]], "*")
      assert([[:text, "-{{br}}"]], "-")
      assert([[:text, "+{{br}}"]], "+")
      assert([[:text, "{t}{{br}}"]], "{t}")
      assert([[:text, "t{{br}}"]], "t")
      assert([[:text, "s{{br}}"], [:text, "t{{br}}"]], "s\nt")
      assert([[:text, "s{{br}}"], [:empty], [:text, "t{{br}}"]], "s\n\nt")

      # test_multiline
      assert([[:text, "s{{br}}"], [:text, "t{{br}}"]], "s\nt")
      assert([[:text, "s{{br}}"], [:text, "t{{br}}"]], "s~\nt~") # no change
      assert([[:text, "s{{br}}"], [:empty], [:text, "t{{br}}"]], "s\n\nt")
      assert([[:text, "s{{br}}"], [:text, "{{br}}"], [:text, "t{{br}}"]],
	     "s~\n~\nt~") # no change

      # test_ref
      assert([[:text, "[[t]]{{br}}"]], "[[t]]")
    end
  end

  class TC_InlineTokenizer < Test::Unit::TestCase
    def assert(e, str)
      tokenizer = Qwik::InlineTokenizer.new
      assert_equal(e, tokenizer.tokenize(str))
    end

    def test_all
      assert([], "")
      assert(["t"], "t")
      assert([:"''", "t", :"''"], "''t''")
      assert([:"'''", "t", :"'''"], "'''t'''")
      assert(["'"], "'")
      assert(["'", "t"], "'t")
      assert([:"==", "t", :"=="], "==t==")
      assert(["="], "=")
      assert(["=", "t"], "=t")
      assert([[:ref, "t"]], "[[t]]")
      assert([[:ref, "s|t"]], "[[s|t]]")
      assert(["]"], "]")
      assert(["[", "t", "]"], "[t]")
      assert([[:plugin, "t"]], "{{t}}")
      assert([[:plugin, "t", "a"]], "{{t(a)}}")
      assert(["{", "t", "}"], "{t}")
      assert([[:url, "http://e.com/"]], "http://e.com/")
      assert(["t ", [:url, "http://e.com/"], " a"], "t http://e.com/ a")
      assert([[:url, "https://e.com/"]], "https://e.com/")
      assert(["a ", :"''", "b", :"''", " ", :"'''", "c", :"'''", " ",
	       :==, "d", :==, " ", [:ref, "e"], " ",
	       [:plugin, "f"], " ", [:url, "http://e.com/"], " g"],
	     "a ''b'' '''c''' ==d== [[e]] {{f}} http://e.com/ g")

      # test_ref
      assert([[:ref, "t"]], "[[t]]")
      assert([[:ref, ".attach"]], "[[.attach]]")
      assert([[:ref, ".attach/t.txt"]], "[[.attach/t.txt]]")
      assert([[:ref, ".attach/s t.txt"]], "[[.attach/s t.txt]]")
    end
  end
end
