;http://www.abcam.com/index.html?pageconfig=datasheet&intAbID=24668

(defclass abcam-products ()
  ((antibody-roots :initarg :antibody-roots :initform nil :accessor antibody-roots)
   (antibody-ids :initarg :antibody-ids :initform nil :accessor antibody-ids)
   (entries :initarg :entries :initform (make-hash-table :test 'equal) :accessor entries)
   ))

(defparameter *abcam* (make-instance 'abcam-products
				     :antibody-roots 
				     (append 
				      (loop for charcode from (char-code #\A) to (char-code #\Z) 
					 collect (code-char charcode))
				      (loop for charcode from (char-code #\0) to (char-code #\9) 
					 collect (code-char charcode)))))

(defmethod cache-index-pages ((a abcam-products))
  (let ((ids (make-hash-table :test 'equal)))
    (loop for char in (antibody-roots a)
       for url = (format nil "http://www.abcam.com/index.html?pageconfig=catalog_byproducttype&intProductTypeID=1&strStartChar=~a" char)
       do
       (let* ((firstpage (get-url url :persist t :verbose t))
	      (otherpages (mapcar 'car (all-matches firstpage "(?s)(?i)intResultsPage=(\\d+)" 1))))
	 (dolist (id (all-matches firstpage "(?s)(?i)ab\\d+" 0))
	   (setf (gethash (car id) ids) t))
	 (loop for pagenum in otherpages
	    for url = (format nil  "http://www.abcam.com/index.html?pageconfig=catalog_byproducttype&intProductTypeID=1&strStartChar=~a&intResultsPage=~a&tr=59"
			      char pagenum)
	    for page = (get-url url :persist t :verbose t)
	    do 
	      (dolist (id (all-matches page "(?s)(?i)ab\\d+" 0))
		(setf (gethash (car id) ids) t))
	    )))
    ids))

(defmethod antibody-ids :around ((a abcam-products))
  (or (call-next-method)
      (let ((them nil))
	(maphash (lambda(id _) 
		   (declare (ignore _))
		   (assert (stringp id) () "oops ~a" id)
		   (push (regex-replace-all "^ab" id "") them))
		 (setq @ (cache-index-pages a)))
	(setf (antibody-ids a) them)
	them)))

(defvar *stopit* nil)

;; ssh -L 8080:www.abcam.com:80 alanr@mumble.net
(defmethod cache-datasheet-pages ((a abcam-products))
  (loop for id in (antibody-ids a)
     for url =  (format nil "http://www.abcam.com/index.html?datasheet=~a" id)
     do 
       (unless (probe-file (url-cached-file-name url))
	 (get-url url :dont-cache t :persist t :verbose t ));:tunnel "localhost:8080"))
       (sleep .001)))

(defmethod each-datasheet-page ((a abcam-products) f)
  (loop for id in (antibody-ids a)
     for url =  (format nil "http://www.abcam.com/index.html?datasheet=~a" id)
     do 
       (funcall f id (get-url url :dont-cache t :nofetch t))
       (sleep .001)))


(defmethod parse-datasheet-page ((a abcam-products) page)
  (let* ((datasheet (caar (all-matches page "(?s)<DATASHEET>(.*)</DATASHEET>" 1)))
	 (inner-table (caar (all-matches datasheet "(?s)(?i)<table.*?>(.*)</table>" 1)))
	 (inner-table2 (caar (all-matches inner-table "(?s)(?i)<table.*?>(.*)</table>" 1)))
	 (database-links (and (#"matches" inner-table2 "(?s)(?i).*<b>Database links</b>.*")
			      (mapcar 'car (all-matches (caar (all-matches inner-table2
							      "(?s)(?i)<b>Database links</b>(.*)</td></tr></table>" 1))
						   "(?s)(?i)<a href=\"([^\"]*)\""
					   1))))
	 (pmids (mapcar 'car (all-matches datasheet "http://www\\.ncbi\\.nlm\\.nih\\.gov/entrez/query\\.fcgi\\?cmd=Retrieve&db=PubMed&list_uids=(\\d+)&dopt=Abstract" 1)))
	 (noise-removed  (regex-replace-all "(?s)(?i)<(table|select|script).*?>.*?</\\1>" inner-table2 ""))
	 (table-rows (mapcar 'car (all-matches noise-removed "(?s)(?i)<tr.*?>\\s*(.*?)\\s*</tr>" 1)))
	 (key-values (loop for row in table-rows 
			append
			(all-matches row "(?s)(?i)<b>([^<].*?)</b>.*?<td.*?>\\s*(<p>){0,1}\\s*(.*?)\\s*(</p>){0,1}\\s*</td>" 1 3))))
    (setq key-values
	  (loop for (key value) in key-values
	       do (setq value (regex-replace-all "<a href.*?>(.*?)</a>" value "$1"))
	     collect
	     (cond ((equalp key "Research areas")
		    (list key (remove-if (lambda(el) (#"matches" el "(^&.*$)|(^\\s*$)")) (split-at-regex value "\\s*<.*?>\\s*"))))
		   (t (list key value)))))
    (when database-links
      (setf (cdr (assoc "Database links" key-values :test 'equal))
	    database-links))
    ;; (print-db 
    ;;      (length page)
    ;;      (length datasheet)
    ;;      (length inner-table)
    ;;      (length inner-table2)
    ;;      (length noise-removed)
    ;;      (length table-rows)
    ;;      )
    (if pmids
	(append key-values (list (list* "pmids" pmids)) )
	key-values)))

(defmethod parse-all-remaining ((a abcam-products))
  (loop for id in (antibody-ids a)
     do 
       (unless (gethash id (entries a))
	 (sleep .001)
	 (format t "[~a]" id)
	 (multiple-value-bind (value errorp)
	     (ignore-errors 
	       (setf (gethash id (entries a)) 
		     (parse-datasheet-page a (get-url (format nil "http://www.abcam.com/index.html?datasheet=~a" id) :dont-cache t :nofetch t))))
	   (when errorp (format t "~%Error in ~a~%" id ))))))
       
(defmethod dump-entries ((c abcam-products) path)
  (with-open-file (f path :if-does-not-exist :create :if-exists :supersede :direction :output)
    (maphash (lambda(k v) (unless (assoc :id v)
			    (setq v (cons (list :id k) v)))
		    (pprint v f))
	     (entries c))))

(defmethod retrieve-entries ((c abcam-products) path)
  (with-open-file (f path :direction :input)
    (loop for entry = (read f nil :eof)
	 until (eq entry :eof)
	 do
	 (setf (gethash (second (assoc :id entry)) (entries c)) entry))))

;(dump-entries *abcam* "~/lsw/hcls/biordf/reagents/abcam-products.txt")
;(retrieve-entries *abcam* "~/lsw/hcls/biordf/reagents/abcam-products.txt")



  get rid of <select>.*?</select>
grab title
get rid of comments <-- .* -->

  <div id="submenu"> <div id="submenuItemsDatasheet"> <div id="catalogBread"><p>
Products: <a href="index.html?c=246&t0=5769547-99119039">Cell Biology</a> <span class="breadDelim">&gt;&gt;</span> <a href="index.html?c=252&t0=5769547-99119039">Microbiology</a> <span class="breadDelim">&gt;&gt;</span> <a href="index.html?c=630&t0=5769547-99119039">Eukaryotic Microorganisms</a> <span class="breadDelim">&gt;&gt;</span> <a href="index.html?c=638&t0=5769547-99119039">Parasites</a> <span class="breadDelim">&gt;&gt;</span> <a href="index.html?c=666&t0=5769547-99119039">Plasmodium spp</a></p></div> <h1>Plasmodium falciparum antibody [MPFG-55P] (HRP) (ab30384) datasheet</h1> <table border="0" cellspacing="0" cellpadding="0"> <tr> <td><span class="ends">&nbsp;&nbsp;</span></td> <td nowrap><p class="datasheet"><span class="sel"><img src="assets/images/arrow_down.gif" align="absmiddle"> Datasheet</span><span class="sep">&nbsp;&nbsp;&nbsp;</span><span class="unsel"><img src="assets/images/arrow_down_offw.gif" align="absmiddle"> <a href="/index.html?pageconfig=reviews&intAbID=30384&t0=5769547-99119039">Abreviews&trade;</a></span><span class="sep">&nbsp;&nbsp;&nbsp;</span><span class="unsel"><img src="assets/images/arrow_down_offw.gif" align="absmiddle"> <a href="/index.html?pageconfig=technical&mode=enquiries&intAbID=30384&reset=&t0=5769547-99119039">Enquiries</a></span><span class="sep">&nbsp;&nbsp;&nbsp;</span><span class="unsel"><img src="assets/images/arrow_down_offw.gif" align="absmiddle"> <a href="/index.html?pageconfig=protocols&intAbID=30384&t0=5769547-99119039">Protocols</a></span><span class="sep">&nbsp;&nbsp;&nbsp;</span><span class="unsel"><img src="assets/images/arrow_down_offw.gif" align="absmiddle"> <a href="/index.html?pageconfig=related&intAbID=30384&t0=5769547-99119039">Related products</a></span></p></td> <td style="border-bottom:1px solid gray" width="100%">&nbsp;&nbsp;</td> </tr> </table> </div> </div>  


<DATASHEET>
</DATASHEET>

two top level tables. inner is content. then small inner tables to ignore
ignore stuff below 
<tr> <td colspan="3"> <hr> <h2>Plasmodium falciparum antibody [MPFG-55P] (HRP) - related products:</h2> </td> </tr> 

;(cached-url-safari "http://www.abcam.com/index.html?datasheet=30384")