Hacking Tricks

otc scrapper new

<p> <span style="background-color: #1f1f1f; color: #c586c0; font-family: Menlo, Monaco, "Courier New", monospace; font-size: 12px; white-space: pre;">from</span><span style="background-color: #1f1f1f; color: #cccccc; font-family: Menlo, Monaco, "Courier New", monospace; font-size: 12px; white-space: pre;"> </span><span style="background-color: #1f1f1f; color: #4ec9b0; font-family: Menlo, Monaco, "Courier New", monospace; font-size: 12px; white-space: pre;">bs4</span><span style="background-color: #1f1f1f; color: #cccccc; font-family: Menlo, Monaco, "Courier New", monospace; font-size: 12px; white-space: pre;"> </span><span style="background-color: #1f1f1f; color: #c586c0; font-family: Menlo, Monaco, "Courier New", monospace; font-size: 12px; white-space: pre;">import</span><span style="background-color: #1f1f1f; color: #cccccc; font-family: Menlo, Monaco, "Courier New", monospace; font-size: 12px; white-space: pre;"> </span><span style="background-color: #1f1f1f; color: #4ec9b0; font-family: Menlo, Monaco, "Courier New", monospace; font-size: 12px; white-space: pre;">BeautifulSoup</span></p><div style="background-color: #1f1f1f; color: #cccccc; font-family: Menlo, Monaco, "Courier New", monospace; font-size: 12px; line-height: 18px; white-space: pre;"><div><span style="color: #c586c0;">import</span> <span style="color: #4ec9b0;">requests</span></div><div><span style="color: #c586c0;">import</span> <span style="color: #4ec9b0;">pandas</span> <span style="color: #c586c0;">as</span> <span style="color: #4ec9b0;">pd</span></div><div><span style="color: #c586c0;">from</span> <span style="color: #4ec9b0;">bs4</span> <span style="color: #c586c0;">import</span> <span style="color: #4ec9b0;">Tag</span></div><div><span style="color: #c586c0;">import</span> <span style="color: #4ec9b0;">time</span></div><br /><div><span style="color: #9cdcfe;">headers</span> <span style="color: #d4d4d4;">=</span> {</div><div>    <span style="color: #ce9178;">'Accept-Encoding'</span>: <span style="color: #ce9178;">'gzip, deflate, sdch'</span>,</div><div>    <span style="color: #ce9178;">'Accept-Language'</span>: <span style="color: #ce9178;">'en-US,en;q=0.8'</span>,</div><div>    <span style="color: #ce9178;">'Upgrade-Insecure-Requests'</span>: <span style="color: #ce9178;">'1'</span>,</div><div>    <span style="color: #ce9178;">'User-Agent'</span>: <span style="color: #ce9178;">'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'</span>,</div><div>    <span style="color: #ce9178;">'Accept'</span>: <span style="color: #ce9178;">'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'</span>,</div><div>    <span style="color: #ce9178;">'Cache-Control'</span>: <span style="color: #ce9178;">'max-age=0'</span>,</div><div>    <span style="color: #ce9178;">'Connection'</span>: <span style="color: #ce9178;">'keep-alive'</span>,</div><div>}</div><br /><br /><div><span style="color: #9cdcfe;">url</span> <span style="color: #d4d4d4;">=</span> <span style="color: #ce9178;">"https://www.1mg.com/otc/small-wonder-admire-baby-feeding-bottle-small-green-otc499746?wpsrc=Google+Organic+Search"</span></div><div><span style="color: #9cdcfe;">source</span> <span style="color: #d4d4d4;">=</span> <span style="color: #4ec9b0;">requests</span>.<span style="color: #dcdcaa;">get</span> (<span style="color: #9cdcfe;">url</span>, <span style="color: #9cdcfe;">allow_redirects</span><span style="color: #d4d4d4;">=</span><span style="color: #569cd6;">False</span>, <span style="color: #9cdcfe;">headers</span><span style="color: #d4d4d4;">=</span><span style="color: #9cdcfe;">headers</span>)</div><div><span style="color: #9cdcfe;">soup</span> <span style="color: #d4d4d4;">=</span> <span style="color: #4ec9b0;">BeautifulSoup</span>(<span style="color: #9cdcfe;">source</span>.<span style="color: #9cdcfe;">text</span>,<span style="color: #ce9178;">'html.parser'</span>)</div><div><span style="color: #9cdcfe;">title</span> <span style="color: #d4d4d4;">=</span> <span style="color: #9cdcfe;">soup</span>.<span style="color: #dcdcaa;">find</span>(<span style="color: #ce9178;">'h1'</span>, <span style="color: #9cdcfe;">class_</span><span style="color: #d4d4d4;">=</span><span style="color: #ce9178;">"ProductTitle__product-title___3QMYH"</span>)</div><div><span style="color: #9cdcfe;">maindiv</span> <span style="color: #d4d4d4;">=</span> <span style="color: #9cdcfe;">soup</span>.<span style="color: #dcdcaa;">find</span>(<span style="color: #ce9178;">'div'</span>, <span style="color: #9cdcfe;">class_</span><span style="color: #d4d4d4;">=</span><span style="color: #ce9178;">"ProductDescription__description-content___A_qCZ"</span>)</div><br /><div><span style="color: #9cdcfe;">first_strong</span> <span style="color: #d4d4d4;">=</span> <span style="color: #9cdcfe;">maindiv</span>.<span style="color: #dcdcaa;">find</span>(<span style="color: #ce9178;">'strong'</span>)</div><div><span style="color: #c586c0;">if</span> <span style="color: #9cdcfe;">first_strong</span> <span style="color: #569cd6;">is</span> <span style="color: #569cd6;">None</span>:</div><div>    <span style="color: #9cdcfe;">first_strong</span> <span style="color: #d4d4d4;">=</span> <span style="color: #9cdcfe;">maindiv</span>.<span style="color: #dcdcaa;">find</span>(<span style="color: #ce9178;">'b'</span>)</div><div>    <span style="color: #9cdcfe;">tag_used</span> <span style="color: #d4d4d4;">=</span> <span style="color: #ce9178;">'b'</span></div><div><span style="color: #c586c0;">else</span>:</div><div>    <span style="color: #9cdcfe;">tag_used</span> <span style="color: #d4d4d4;">=</span> <span style="color: #ce9178;">'strong'</span></div><br /><br /><br /><div><span style="color: #6a9955;">#------------------------------------USES</span></div><div><span style="color: #c586c0;">if</span> <span style="color: #9cdcfe;">first_strong</span>:</div><div>    <span style="color: #9cdcfe;">second_strong</span> <span style="color: #d4d4d4;">=</span> <span style="color: #9cdcfe;">first_strong</span>.<span style="color: #dcdcaa;">find_next</span>(<span style="color: #9cdcfe;">tag_used</span>).<span style="color: #dcdcaa;">find_next</span>(<span style="color: #9cdcfe;">tag_used</span>)</div><div>    <span style="color: #c586c0;">if</span> <span style="color: #9cdcfe;">second_strong</span>:</div><div>        <span style="color: #c586c0;">if</span> <span style="color: #9cdcfe;">second_strong</span>.<span style="color: #9cdcfe;">text</span> <span style="color: #569cd6;">in</span> [<span style="color: #ce9178;">"Key Ingredients:"</span>, <span style="color: #ce9178;">"Key Benefits:"</span>, <span style="color: #ce9178;">"Directions For Use:"</span>, <span style="color: #ce9178;">"Safety Information:"</span>, <span style="color: #ce9178;">"Product Specifications and Features:"</span>,</div><div>                                  <span style="color: #ce9178;">"Key Ingredients"</span>, <span style="color: #ce9178;">"Key Ingredient:"</span>, <span style="color: #ce9178;">"Key Benefits"</span>, <span style="color: #ce9178;">"Directions For Use"</span>, <span style="color: #ce9178;">"Safety Information"</span>, <span style="color: #ce9178;">"Product Specifications and Features"</span>,</div><div>                                  <span style="color: #ce9178;">"Brief Details:"</span>, <span style="color: #ce9178;">"Brief Details"</span>, <span style="color: #ce9178;">"Basic Details:"</span>, <span style="color: #ce9178;">"Basic Details"</span>]:</div><div>            <span style="color: #9cdcfe;">first_strong_index</span> <span style="color: #d4d4d4;">=</span> <span style="color: #9cdcfe;">maindiv</span>.<span style="color: #9cdcfe;">contents</span>.<span style="color: #dcdcaa;">index</span>(<span style="color: #9cdcfe;">first_strong</span>)</div><div>            <span style="color: #9cdcfe;">second_strong_index</span> <span style="color: #d4d4d4;">=</span> <span style="color: #9cdcfe;">maindiv</span>.<span style="color: #9cdcfe;">contents</span>.<span style="color: #dcdcaa;">index</span>(<span style="color: #9cdcfe;">second_strong</span>)</div><div>            <span style="color: #9cdcfe;">result_html</span> <span style="color: #d4d4d4;">=</span> <span style="color: #9cdcfe;">maindiv</span>.<span style="color: #9cdcfe;">contents</span>[<span style="color: #9cdcfe;">first_strong_index</span> <span style="color: #d4d4d4;">+</span> <span style="color: #b5cea8;">2</span>:<span style="color: #9cdcfe;">second_strong_index</span>]</div><div>            <span style="color: #c586c0;">if</span> <span style="color: #dcdcaa;">any</span>(<span style="color: #9cdcfe;">tag</span>.name <span style="color: #d4d4d4;">==</span> <span style="color: #9cdcfe;">tag_used</span> <span style="color: #c586c0;">and</span> <span style="color: #ce9178;">"Uses"</span> <span style="color: #c586c0;">in</span> <span style="color: #9cdcfe;">tag</span>.<span style="color: #9cdcfe;">text</span> <span style="color: #c586c0;">for</span> <span style="color: #9cdcfe;">tag</span> <span style="color: #c586c0;">in</span> <span style="color: #9cdcfe;">result_html</span>):<span style="color: #6a9955;">#agar uses milega isko <b> ke ya <strong> ke baad to hi work krega ye</span></div><div>                <span style="color: #9cdcfe;">result_html</span> <span style="color: #d4d4d4;">=</span> [<span style="color: #9cdcfe;">item</span> <span style="color: #c586c0;">for</span> <span style="color: #9cdcfe;">item</span> <span style="color: #c586c0;">in</span> <span style="color: #9cdcfe;">result_html</span> <span style="color: #c586c0;">if</span> <span style="color: #569cd6;">not</span> (<span style="color: #dcdcaa;">isinstance</span>(<span style="color: #9cdcfe;">item</span>, <span style="color: #4ec9b0;">Tag</span>) <span style="color: #569cd6;">and</span> <span style="color: #ce9178;">'Uses'</span> <span style="color: #569cd6;">in</span> <span style="color: #9cdcfe;">item</span>.<span style="color: #9cdcfe;">text</span>)]</div><div>                <span style="color: #c586c0;">if</span> <span style="color: #569cd6;">not</span> <span style="color: #dcdcaa;">any</span>(<span style="color: #9cdcfe;">tag</span>.<span style="color: #9cdcfe;">name</span> <span style="color: #d4d4d4;">==</span> <span style="color: #ce9178;">'ul'</span> <span style="color: #c586c0;">or</span> <span style="color: #9cdcfe;">tag</span>.<span style="color: #9cdcfe;">name</span> <span style="color: #d4d4d4;">==</span> <span style="color: #ce9178;">'li'</span> <span style="color: #c586c0;">for</span> <span style="color: #9cdcfe;">tag</span> <span style="color: #c586c0;">in</span> <span style="color: #9cdcfe;">result_html</span>):<span style="color: #6a9955;">#Agar hamare result me ul and li tags honge to us case me error dega.</span></div><div>                    <span style="color: #9cdcfe;">result_html</span> <span style="color: #d4d4d4;">=</span> [<span style="color: #9cdcfe;">i</span>.<span style="color: #9cdcfe;">text</span>.<span style="color: #dcdcaa;">strip</span>() <span style="color: #c586c0;">for</span> <span style="color: #9cdcfe;">i</span> <span style="color: #c586c0;">in</span> <span style="color: #9cdcfe;">result_html</span> <span style="color: #c586c0;">if</span> <span style="color: #9cdcfe;">i</span> <span style="color: #d4d4d4;">!=</span> <span style="color: #ce9178;">' '</span> <span style="color: #569cd6;">and</span> <span style="color: #9cdcfe;">i</span> <span style="color: #d4d4d4;">!=</span> <span style="color: #ce9178;">'</span><span style="color: #d7ba7d;">\n</span><span style="color: #ce9178;">'</span>]</div><div>                    <span style="color: #9cdcfe;">result_html</span> <span style="color: #d4d4d4;">=</span> <span style="color: #ce9178;">''</span>.<span style="color: #dcdcaa;">join</span>(<span style="color: #9cdcfe;">result_html</span>)</div><div>                    </div><div>                    <span style="color: #9cdcfe;">result</span> <span style="color: #d4d4d4;">=</span> <span style="color: #ce9178;">""</span>.<span style="color: #dcdcaa;">join</span>([<span style="color: #4ec9b0;">str</span>(<span style="color: #9cdcfe;">content</span>) <span style="color: #c586c0;">for</span> <span style="color: #9cdcfe;">content</span> <span style="color: #c586c0;">in</span> <span style="color: #9cdcfe;">result_html</span>])</div><div>                    <span style="color: #9cdcfe;">finalresult</span> <span style="color: #d4d4d4;">=</span> <span style="color: #ce9178;">""</span><span style="color: #d4d4d4;">+</span><span style="color: #9cdcfe;">result</span>.<span style="color: #dcdcaa;">replace</span>(<span style="color: #ce9178;">"<br>"</span>, <span style="color: #ce9178;">""</span>).<span style="color: #dcdcaa;">replace</span>(<span style="color: #ce9178;">"<br/>"</span>, <span style="color: #ce9178;">""</span>)<span style="color: #d4d4d4;">+</span><span style="color: #ce9178;">""</span></div><div>                    <span style="color: #c586c0;">try</span>:</div><div>                        <span style="color: #9cdcfe;">descriptionlen</span> <span style="color: #d4d4d4;">=</span> <span style="color: #dcdcaa;">len</span>(<span style="color: #9cdcfe;">finalresult</span>.<span style="color: #dcdcaa;">split</span>())<span style="color: #6a9955;">#Working</span></div><div>                    <span style="color: #c586c0;">except</span>:</div><div>                        <span style="color: #9cdcfe;">descriptionlen</span> <span style="color: #d4d4d4;">=</span> <span style="color: #ce9178;">"Not Found"</span></div><div>                    <span style="color: #dcdcaa;">print</span>(<span style="color: #9cdcfe;">finalresult</span>)</div><div>                    <span style="color: #dcdcaa;">print</span>(<span style="color: #ce9178;">"-------------------"</span>)</div><br /><br /><br /><br /><br /><div><span style="color: #6a9955;">#-------------------------------DESCRIPTION---------</span></div><div><span style="color: #c586c0;">if</span> <span style="color: #9cdcfe;">first_strong</span>:</div><div>    <span style="color: #9cdcfe;">second_strong</span> <span style="color: #d4d4d4;">=</span> <span style="color: #9cdcfe;">first_strong</span>.<span style="color: #dcdcaa;">find_next</span>(<span style="color: #9cdcfe;">tag_used</span>)</div><div>    <span style="color: #c586c0;">if</span> <span style="color: #9cdcfe;">second_strong</span>:</div><div>        <span style="color: #c586c0;">if</span> <span style="color: #9cdcfe;">second_strong</span>.<span style="color: #9cdcfe;">text</span> <span style="color: #569cd6;">in</span> [<span style="color: #ce9178;">"Key Ingredients:"</span>, <span style="color: #ce9178;">"Key Benefits:"</span>, <span style="color: #ce9178;">"Directions For Use:"</span>, <span style="color: #ce9178;">"Safety Information:"</span>, <span style="color: #ce9178;">"Uses:"</span>, <span style="color: #ce9178;">"Product Specifications and Features:"</span>,</div><div>                                  <span style="color: #ce9178;">"Key Ingredients"</span>, <span style="color: #ce9178;">"Key Ingredient:"</span>, <span style="color: #ce9178;">"Key Benefits"</span>, <span style="color: #ce9178;">"Directions For Use"</span>, <span style="color: #ce9178;">"Safety Information"</span>, <span style="color: #ce9178;">"Uses"</span>, <span style="color: #ce9178;">"Product Specifications and Features"</span>,</div><div>                                  <span style="color: #ce9178;">"Brief Details:"</span>, <span style="color: #ce9178;">"Brief Details"</span>, <span style="color: #ce9178;">"Basic Details:"</span>, <span style="color: #ce9178;">"Basic Details"</span>,</div><div>                                  <span style="color: #ce9178;">"Product Specification and Features"</span>, <span style="color: #ce9178;">"Product Specification and Features:"</span>]:</div><div>            <span style="color: #9cdcfe;">first_strong_index</span> <span style="color: #d4d4d4;">=</span> <span style="color: #9cdcfe;">maindiv</span>.<span style="color: #9cdcfe;">contents</span>.<span style="color: #dcdcaa;">index</span>(<span style="color: #9cdcfe;">first_strong</span>)</div><div>            <span style="color: #9cdcfe;">second_strong_index</span> <span style="color: #d4d4d4;">=</span> <span style="color: #9cdcfe;">maindiv</span>.<span style="color: #9cdcfe;">contents</span>.<span style="color: #dcdcaa;">index</span>(<span style="color: #9cdcfe;">second_strong</span>)</div><div>            <span style="color: #9cdcfe;">result_html</span> <span style="color: #d4d4d4;">=</span> <span style="color: #9cdcfe;">maindiv</span>.<span style="color: #9cdcfe;">contents</span>[<span style="color: #9cdcfe;">first_strong_index</span> <span style="color: #d4d4d4;">+</span> <span style="color: #b5cea8;">1</span>:<span style="color: #9cdcfe;">second_strong_index</span>]</div><div>            <span style="color: #c586c0;">if</span> <span style="color: #569cd6;">not</span> <span style="color: #dcdcaa;">any</span>(<span style="color: #9cdcfe;">tag</span>.name <span style="color: #d4d4d4;">==</span> <span style="color: #ce9178;">'ul'</span> <span style="color: #c586c0;">or</span> <span style="color: #9cdcfe;">tag</span>.name <span style="color: #d4d4d4;">==</span> <span style="color: #ce9178;">'li'</span> <span style="color: #c586c0;">for</span> <span style="color: #9cdcfe;">tag</span> <span style="color: #c586c0;">in</span> <span style="color: #9cdcfe;">result_html</span>):</div><div>                <span style="color: #9cdcfe;">result_html</span> <span style="color: #d4d4d4;">=</span> [<span style="color: #9cdcfe;">i</span>.<span style="color: #9cdcfe;">text</span>.<span style="color: #dcdcaa;">strip</span>() <span style="color: #c586c0;">for</span> <span style="color: #9cdcfe;">i</span> <span style="color: #c586c0;">in</span> <span style="color: #9cdcfe;">result_html</span> <span style="color: #c586c0;">if</span> <span style="color: #9cdcfe;">i</span> <span style="color: #d4d4d4;">!=</span> <span style="color: #ce9178;">' '</span> <span style="color: #569cd6;">and</span> <span style="color: #9cdcfe;">i</span> <span style="color: #d4d4d4;">!=</span> <span style="color: #ce9178;">'</span><span style="color: #d7ba7d;">\n</span><span style="color: #ce9178;">'</span>]</div><div>                <span style="color: #9cdcfe;">result_html</span> <span style="color: #d4d4d4;">=</span> <span style="color: #ce9178;">''</span>.<span style="color: #dcdcaa;">join</span>(<span style="color: #9cdcfe;">result_html</span>)</div><br /><div>                <span style="color: #9cdcfe;">result</span> <span style="color: #d4d4d4;">=</span> <span style="color: #ce9178;">""</span>.<span style="color: #dcdcaa;">join</span>([<span style="color: #4ec9b0;">str</span>(<span style="color: #9cdcfe;">content</span>) <span style="color: #c586c0;">for</span> <span style="color: #9cdcfe;">content</span> <span style="color: #c586c0;">in</span> <span style="color: #9cdcfe;">result_html</span>])</div><div>                <span style="color: #9cdcfe;">finalresult</span> <span style="color: #d4d4d4;">=</span> <span style="color: #ce9178;">""</span><span style="color: #d4d4d4;">+</span><span style="color: #9cdcfe;">first_strong</span>.<span style="color: #9cdcfe;">text</span><span style="color: #d4d4d4;">+</span><span style="color: #ce9178;">" "</span><span style="color: #d4d4d4;">+</span><span style="color: #9cdcfe;">result</span>.<span style="color: #dcdcaa;">replace</span>(<span style="color: #ce9178;">"<br>"</span>, <span style="color: #ce9178;">""</span>).<span style="color: #dcdcaa;">replace</span>(<span style="color: #ce9178;">"<br/>"</span>, <span style="color: #ce9178;">""</span>)<span style="color: #d4d4d4;">+</span><span style="color: #ce9178;">""</span></div><div>                <span style="color: #c586c0;">try</span>:</div><div>                    <span style="color: #9cdcfe;">descriptionlen</span> <span style="color: #d4d4d4;">=</span> <span style="color: #dcdcaa;">len</span>(<span style="color: #9cdcfe;">finalresult</span>.<span style="color: #dcdcaa;">split</span>())<span style="color: #6a9955;">#Working</span></div><div>                <span style="color: #c586c0;">except</span>:</div><div>                    <span style="color: #9cdcfe;">descriptionlen</span> <span style="color: #d4d4d4;">=</span> <span style="color: #ce9178;">"Not Found"</span></div><div>                <span style="color: #dcdcaa;">print</span>(<span style="color: #9cdcfe;">finalresult</span>)</div><div>                <span style="color: #dcdcaa;">print</span>(<span style="color: #ce9178;">"-------------------"</span>)</div><br /><br /><br /><br /><br /><br /><br /><div><span style="color: #6a9955;">#---------------------------------------------------------------------------------------------------------------------------</span></div><br /><div>                <span style="color: #c586c0;">try</span>:</div><div>                    <span style="color: #4fc1ff;">BOLD_TAGS</span> <span style="color: #d4d4d4;">=</span> [</div><div>                        <span style="color: #ce9178;">"Product Specification and Features"</span>,</div><div>                        <span style="color: #ce9178;">"Product Specifications and Features"</span>,</div><div>                        <span style="color: #ce9178;">"Product Specifications & Features"</span>,</div><div>                        <span style="color: #ce9178;">"Product Specification & Feature"</span></div><div>                    ]</div><br /><div>                    <span style="color: #9cdcfe;">benefits</span> <span style="color: #d4d4d4;">=</span> <span style="color: #569cd6;">None</span>  <span style="color: #6a9955;"># Initialize benefits outside the loop to avoid scoping issues</span></div><br /><div>                    <span style="color: #c586c0;">for</span> <span style="color: #4fc1ff;">BOLD_TAG</span> <span style="color: #c586c0;">in</span> <span style="color: #4fc1ff;">BOLD_TAGS</span>:</div><div>                        <span style="color: #c586c0;">try</span>:</div><div>                            <span style="color: #9cdcfe;">benefits</span> <span style="color: #d4d4d4;">=</span> <span style="color: #9cdcfe;">maindiv</span>.<span style="color: #dcdcaa;">find</span>(<span style="color: #9cdcfe;">tag_used</span>, <span style="color: #9cdcfe;">text</span><span style="color: #d4d4d4;">=</span><span style="color: #4fc1ff;">BOLD_TAG</span>).<span style="color: #dcdcaa;">find_next_sibling</span>(<span style="color: #ce9178;">"ul"</span>).<span style="color: #dcdcaa;">find_all</span>(<span style="color: #ce9178;">"li"</span>)</div><div>                            <span style="color: #4fc1ff;">BOLD_TAG</span> <span style="color: #d4d4d4;">=</span> <span style="color: #4fc1ff;">BOLD_TAG</span></div><div>                            <span style="color: #c586c0;">break</span>  <span style="color: #6a9955;"># Exit the loop if benefits are found</span></div><div>                        <span style="color: #c586c0;">except</span> <span style="color: #4ec9b0;">AttributeError</span>:</div><div>                            <span style="color: #c586c0;">pass</span>  <span style="color: #6a9955;"># Ignore the exception and continue to the next BOLD_TAG</span></div><br /><div>                        <span style="color: #c586c0;">try</span>:</div><div>                            <span style="color: #9cdcfe;">benefits</span> <span style="color: #d4d4d4;">=</span> <span style="color: #9cdcfe;">maindiv</span>.<span style="color: #dcdcaa;">find</span>(<span style="color: #9cdcfe;">tag_used</span>, <span style="color: #9cdcfe;">text</span><span style="color: #d4d4d4;">=</span><span style="color: #4fc1ff;">BOLD_TAG</span> <span style="color: #d4d4d4;">+</span> <span style="color: #ce9178;">":"</span>).<span style="color: #dcdcaa;">find_next_sibling</span>(<span style="color: #ce9178;">"ul"</span>).<span style="color: #dcdcaa;">find_all</span>(<span style="color: #ce9178;">"li"</span>)</div><div>                            <span style="color: #4fc1ff;">BOLD_TAG</span> <span style="color: #d4d4d4;">=</span> <span style="color: #4fc1ff;">BOLD_TAG</span> <span style="color: #d4d4d4;">+</span> <span style="color: #ce9178;">":"</span>  <span style="color: #6a9955;"># Update BOLD_TAG if colon was added</span></div><div>                            <span style="color: #c586c0;">break</span>  <span style="color: #6a9955;"># Exit the loop if benefits are found</span></div><div>                        <span style="color: #c586c0;">except</span> <span style="color: #4ec9b0;">AttributeError</span>:</div><div>                            <span style="color: #c586c0;">pass</span>  <span style="color: #6a9955;"># Ignore the exception and continue to the next BOLD_TAG</span></div><div>                    </div><div>                    </div><div>                    <span style="color: #c586c0;">try</span>:</div><div>                        <span style="color: #9cdcfe;">next_header_tag</span> <span style="color: #d4d4d4;">=</span> <span style="color: #9cdcfe;">maindiv</span>.<span style="color: #dcdcaa;">find</span>(<span style="color: #9cdcfe;">tag_used</span>, <span style="color: #9cdcfe;">text</span><span style="color: #d4d4d4;">=</span><span style="color: #4fc1ff;">BOLD_TAG</span>).<span style="color: #dcdcaa;">find_next_sibling</span>(<span style="color: #9cdcfe;">tag_used</span>)</div><div>                        <span style="color: #c586c0;">if</span> <span style="color: #9cdcfe;">next_header_tag</span>:</div><div>                            <span style="color: #9cdcfe;">next_header</span> <span style="color: #d4d4d4;">=</span> <span style="color: #9cdcfe;">next_header_tag</span>.<span style="color: #9cdcfe;">text</span></div><div>                        <span style="color: #c586c0;">else</span>:</div><div>                            <span style="color: #9cdcfe;">next_header</span> <span style="color: #d4d4d4;">=</span> <span style="color: #9cdcfe;">maindiv</span>.<span style="color: #dcdcaa;">find</span>(<span style="color: #9cdcfe;">tag_used</span>, <span style="color: #9cdcfe;">text</span><span style="color: #d4d4d4;">=</span><span style="color: #4fc1ff;">BOLD_TAG</span>).<span style="color: #dcdcaa;">find_next_sibling</span>(<span style="color: #ce9178;">"div"</span>)</div><div>                    <span style="color: #c586c0;">except</span>:</div><div>                        <span style="color: #9cdcfe;">next_header</span> <span style="color: #d4d4d4;">=</span> <span style="color: #569cd6;">None</span></div><div>                        </div><div>                        </div><div>                    <span style="color: #9cdcfe;">benefits</span> <span style="color: #d4d4d4;">=</span> [<span style="color: #9cdcfe;">benefit</span>.text.strip() <span style="color: #c586c0;">for</span> <span style="color: #9cdcfe;">benefit</span> <span style="color: #c586c0;">in</span> <span style="color: #9cdcfe;">benefits</span>]</div><div>                    <span style="color: #9cdcfe;">benefits_header</span> <span style="color: #d4d4d4;">=</span> <span style="color: #9cdcfe;">soup</span>.<span style="color: #dcdcaa;">find</span>(<span style="color: #9cdcfe;">tag_used</span>, <span style="color: #9cdcfe;">text</span><span style="color: #d4d4d4;">=</span><span style="color: #4fc1ff;">BOLD_TAG</span>)</div><br /><br /><div>                    <span style="color: #c586c0;">if</span> <span style="color: #9cdcfe;">benefits_header</span>:</div><div>                        <span style="color: #9cdcfe;">uls</span> <span style="color: #d4d4d4;">=</span> <span style="color: #9cdcfe;">benefits_header</span>.<span style="color: #dcdcaa;">find_next_siblings</span>(<span style="color: #ce9178;">"ul"</span>)</div><div>                        <span style="color: #9cdcfe;">benefits</span> <span style="color: #d4d4d4;">=</span> []</div><div>                        <span style="color: #c586c0;">for</span> <span style="color: #9cdcfe;">ul</span> <span style="color: #c586c0;">in</span> <span style="color: #9cdcfe;">uls</span>:</div><div>                            <span style="color: #6a9955;"># Check if the previous sibling is a <strong> or <b> tag and contains the text of the next header</span></div><div>                            <span style="color: #c586c0;">if</span> <span style="color: #9cdcfe;">next_header</span> <span style="color: #569cd6;">and</span> <span style="color: #9cdcfe;">ul</span>.<span style="color: #dcdcaa;">find_previous_sibling</span>(<span style="color: #569cd6;">lambda</span> <span style="color: #9cdcfe;">tag</span>: <span style="color: #9cdcfe;">(tag_used)</span> <span style="color: #c586c0;">and</span> <span style="color: #9cdcfe;">next_header</span> <span style="color: #c586c0;">in</span> <span style="color: #9cdcfe;">tag</span>.get_text(<span style="color: #9cdcfe;">strip</span><span style="color: #d4d4d4;">=</span><span style="color: #569cd6;">True</span>)):</div><div>                                <span style="color: #c586c0;">break</span></div><div>                            <span style="color: #6a9955;"># Check if the previous sibling is a <div> tag (your second condition)</span></div><div>                            <span style="color: #c586c0;">elif</span> <span style="color: #9cdcfe;">ul</span>.<span style="color: #dcdcaa;">find_previous_sibling</span>(<span style="color: #ce9178;">"div"</span>):</div><div>                                <span style="color: #c586c0;">break</span></div><div>                            <span style="color: #9cdcfe;">benefits</span>.<span style="color: #dcdcaa;">extend</span>([<span style="color: #9cdcfe;">li</span>.text.replace(<span style="color: #ce9178;">'</span><span style="color: #d7ba7d;">\xa0</span><span style="color: #ce9178;">'</span>, <span style="color: #ce9178;">' '</span>) <span style="color: #c586c0;">for</span> <span style="color: #9cdcfe;">li</span> <span style="color: #c586c0;">in</span> <span style="color: #9cdcfe;">ul</span>.find_all(<span style="color: #ce9178;">"li"</span>)])</div><br /><br /><div>                        <span style="color: #9cdcfe;">word_count</span> <span style="color: #d4d4d4;">=</span> <span style="color: #dcdcaa;">sum</span>([<span style="color: #dcdcaa;">len</span>(<span style="color: #9cdcfe;">benefit</span>.split()) <span style="color: #c586c0;">for</span> <span style="color: #9cdcfe;">benefit</span> <span style="color: #c586c0;">in</span> <span style="color: #9cdcfe;">benefits</span>])</div><div>                        <span style="color: #dcdcaa;">print</span>(<span style="color: #4fc1ff;">BOLD_TAG</span>, <span style="color: #9cdcfe;">benefits</span>)</div><br /><div>                        <span style="color: #dcdcaa;">print</span>(<span style="color: #4fc1ff;">BOLD_TAG</span>, <span style="color: #ce9178;">"Count:-"</span>, <span style="color: #9cdcfe;">word_count</span>)</div><div>                        <span style="color: #dcdcaa;">print</span>(<span style="color: #ce9178;">"-------------------"</span>)</div><div>                        </div><div>                    <span style="color: #c586c0;">else</span>:</div><div>                        <span style="color: #dcdcaa;">print</span>(<span style="color: #4fc1ff;">BOLD_TAG</span>, <span style="color: #ce9178;">" not found"</span>)</div><div>                        <span style="color: #dcdcaa;">print</span>(<span style="color: #ce9178;">"-------------------"</span>)</div><div>                <span style="color: #c586c0;">except</span>:</div><div>                    <span style="color: #dcdcaa;">print</span>(<span style="color: #ce9178;">"Error in "</span>, <span style="color: #4fc1ff;">BOLD_TAG</span>)</div><div>                    <span style="color: #dcdcaa;">print</span>(<span style="color: #ce9178;">"-------------------"</span>)</div><br /></div>

otc scrapper new

from bs4 import BeautifulSoup import requests import pandas as pd from bs4 import Tag import time headers = { 'Accept...