(defparameter *santcruzinfo* (make-hash-table))

(defun product-info (number)
  (or (gethash number *santcruzinfo*)
      (let ((page (get-url (format nil "http://www.scbt.com/catalog/action.lasso?-database=catalog&-layout=web_detail&-response=detail.lasso&-token.order_id=0&-search&-op=eq&catalog_number=sc-~2,'0d#prodinfo" number) :dont-cache t)))
	(when (consp page) (setf (gethash number *santcruzinfo*) page))
	(setf (gethash number *santcruzinfo*)
	      (append (list number)
		      (all-matches page "(?s)<div class=\"path(\\d+).*?>([^>]+?)</a>" 1 2)
		      (car (all-matches page "(?s)<a name=\"prodinfo\">\\s*</a>\\s*<h1><font.*?><b>(.*?): sc-" 1))
		      (list (mapcar 'car (all-matches page "(?s)<a href=\"http://www.ncbi.nlm.nih.gov:80/entrez/query.fcgi\\?cmd=Retrieve&db=PubMed&list_uids=(\\d+)\\s*&dopt=Abstract\"" 1)))
		      (let ((desc (all-matches page "(?s)<div class=\"normal\">\\s*<!-- actual product block, if statements prevent empty bullets -->\\s*<ul>(.*?)</ul>" 1)))
			(and desc
			     (list (mapcar 'car (all-matches (caar desc) "<li>(.*?)</li>" 1)))))
		      )))))

(defun write-santacruz ()
  (with-open-file (f "~/repos/lsw/trunk/hcls/biordf/reagents/santacruz.txt"
		     :direction :output :if-exists :supersede)
    (loop for i from 1
	 for entry = (gethash i *santcruzinfo* :eof)
	 until (eq entry :eof)
	 do 
	 (print entry f))))

(defun pull-all-santacruz (&optional (from 0) (to 43000))
  (loop for i from from to to do (product-info i) (sleep .002) (princ ".") (if (zerop (mod i 100)) (format t "[~a]" i ))))