(defclass upstate-products ()
  ((antibody-roots :initarg :antibody-roots :initform nil :accessor antibody-roots)
   (antibody-ids :initarg :antibody-ids :initform nil :accessor antibody-ids)
   ))

(defparameter *upstate* (make-instance 'upstate-products
				       :antibody-roots 
				       '("http://www.upstate.com/browse/search.asp?adv=1&page=~a&pt=antibody&u1=pt")
				       ))

(defmethod cache-index-pages ((a upstate-products))
  (let ((ids (make-hash-table :test 'equal))
	(root (format nil (car (antibody-roots a)) 1)))
    (let ((max-pages 
	   (caar(all-matches (get-url root :persist t :verbose t)
			"(?s)</select> of (\\d+)" 1))))
      (if max-pages (setq max-pages (parse-integer max-pages)) 1)
      (loop for page from 1 to max-pages
	 for url = (format nil (car (antibody-roots a)) page)
	 do
	 (dolist (id (all-matches (get-url url :persist t :verbose t) "ProductID=(.*?)\"" 1))
	   (setf (gethash (car id) ids) t)))
      ids)))

(defmethod antibody-ids :around ((a upstate-products))
  (or (call-next-method)
      (let ((them nil))
	(maphash (lambda(id _) 
		   (declare (ignore _))
		   (assert (stringp id) () "oops ~a" id)
		   (push id them))
		 (cache-index-pages a))
	(setf (antibody-ids a) them)
	them)))

(defmethod cache-datasheet-pages ((a upstate-products))
  (loop for id in (antibody-ids a)
     for url =  (format nil "http://www.upstate.com/browse/productdetail.asp?ProductID=~a" id)
     do 
       (unless (probe-file (url-cached-file-name url))
	 (get-url url :dont-cache t :persist t :verbose t ))
       (sleep .001)))



(cached-url-safari "http://www.upstate.com/browse/productdetail.asp?ProductID=AB507")