(defvar *biomol-error-pages* '(1000 999 998 997 996 995 994 993
992 991 990 989 988 987 986 985 984 983 982 981 980 979 978 977
976 975 974 973 972 971 970 969 968 967 966 965 964 963 962 961
960 959 958 957 956 955 954 953 952 951 950 949 948 947 946 945
944 943 942 941 940 939 938 937 936 935 934 933 932 931 902 901
900 899 898 897 896 895 894 893 884 883 882 858 800 710 709 708
707 706 705 704 703 702 701 700 699 698 697 696 695 694 693 692
691 690 643 636 595 594 593 592 591 590 589 588 587 586 585 584
583 582 581 580 579 578 577 576 575 574 573 572 571 570 569 568
567 566 565 564 563 562 553 551 545 542 541 540 539 397 396 395
394 393 392 391 390 389 388 387 386 385 380 384 383 381))

(defun products-for-category (category)
  (unless (member category *biomol-error-pages*)
    (let ((page (get-url (format nil "http://www.biomol.com/Online_Catalog/Online_Catalog/Products/36/?categoryId=~a" category))))
      (if (stringp page)
	  (loop 
	     for (url title) in 
	       (all-matches 
		page "class=\"categoryTitle.*\" href=\"(.*?)\">(.*?)</a>" 1 2)
	     collect (list title category (caar (all-matches url "productId=(\\d+)" 1))))
	  (progn
	    (push category *biomol-error-pages*)
	    nil)))))

(defun dump-products-spreadsheet ()
  (with-open-file (f "~/Desktop/biomol.txt" :direction :output :if-exists :supersede)
    (loop for (title category  productid) in 	 
	 (loop for category from 1 to 1000
	    append (products-for-category category))
       when productid do
	 (format f "~a	~a	~a~%" title category productid))))