Revision - 122541b - Issue 21469: Mitigate risk of false positives with robotparser. - origin: https://github.com/python/cpython

Staging
v0.5.1

visit type:

https://github.com/python/cpython

09 December 2020, 11:06:31 UTC

Revision 122541beceeccce4ef8a9bf739c727ccdcbf2f28 authored by Raymond Hettinger on 13 May 2014, 04:56:33 UTC, committed by Raymond Hettinger on 13 May 2014, 04:56:33 UTC

Issue 21469:  Mitigate risk of false positives with robotparser.

* Repair the broken link to norobots-rfc.txt.

* HTTP response codes >= 500 treated as a failed read rather than as a not
found.  Not found means that we can assume the entire site is allowed.  A 5xx
server error tells us nothing.

* A successful read() or parse() updates the mtime (which is defined to be "the
  time the robots.txt file was last fetched").

* The can_fetch() method returns False unless we've had a read() with a 2xx or
4xx response.  This avoids false positives in the case where a user calls
can_fetch() before calling read().

* I don't see any easy way to test this patch without hitting internet
resources that might change or without use of mock objects that wouldn't
provide must reassurance.

1 parent 73308d6

Files
Changes

Permalinks

Tip revision: 122541beceeccce4ef8a9bf739c727ccdcbf2f28 authored by Raymond Hettinger on 13 May 2014, 04:56:33 UTC
Issue 21469: Mitigate risk of false positives with robotparser.

Tip revision: 122541b

make_versioninfo.vcxproj

<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
  <ItemGroup Label="ProjectConfigurations">
    <ProjectConfiguration Include="Debug|Win32">
      <Configuration>Debug</Configuration>
      <Platform>Win32</Platform>
    </ProjectConfiguration>
    <ProjectConfiguration Include="Debug|x64">
      <Configuration>Debug</Configuration>
      <Platform>x64</Platform>
    </ProjectConfiguration>
    <ProjectConfiguration Include="Release|Win32">
      <Configuration>Release</Configuration>
      <Platform>Win32</Platform>
    </ProjectConfiguration>
    <ProjectConfiguration Include="Release|x64">
      <Configuration>Release</Configuration>
      <Platform>x64</Platform>
    </ProjectConfiguration>
  </ItemGroup>
  <PropertyGroup Label="Globals">
    <ProjectGuid>{F0E0541E-F17D-430B-97C4-93ADF0DD284E}</ProjectGuid>
    <RootNamespace>make_versioninfo</RootNamespace>
  </PropertyGroup>
  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
    <ConfigurationType>Application</ConfigurationType>
    <UseOfMfc>false</UseOfMfc>
    <CharacterSet>NotSet</CharacterSet>
  </PropertyGroup>
  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
    <ConfigurationType>Application</ConfigurationType>
    <UseOfMfc>false</UseOfMfc>
    <CharacterSet>MultiByte</CharacterSet>
  </PropertyGroup>
  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
    <ConfigurationType>Application</ConfigurationType>
  </PropertyGroup>
  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
    <ConfigurationType>Application</ConfigurationType>
  </PropertyGroup>
  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
  <ImportGroup Label="ExtensionSettings">
  </ImportGroup>
  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="PropertySheets">
    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
    <Import Project="pyproject.props" />
    <Import Project="debug.props" />
  </ImportGroup>
  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="PropertySheets">
    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
    <Import Project="pyproject.props" />
    <Import Project="release.props" />
  </ImportGroup>
  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
    <Import Project="pyproject.props" />
    <Import Project="x64.props" />
    <Import Project="debug.props" />
  </ImportGroup>
  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
    <Import Project="pyproject.props" />
    <Import Project="x64.props" />
    <Import Project="release.props" />
  </ImportGroup>
  <PropertyGroup Label="UserMacros" />
  <PropertyGroup>
    <_ProjectFileVersion>10.0.40219.1</_ProjectFileVersion>
    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">AllRules.ruleset</CodeAnalysisRuleSet>
    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" />
    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" />
    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">AllRules.ruleset</CodeAnalysisRuleSet>
    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" />
    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" />
    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">AllRules.ruleset</CodeAnalysisRuleSet>
    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" />
    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" />
    <CodeAnalysisRuleSet Condition="'$(Configuration)|$(Platform)'=='Release|x64'">AllRules.ruleset</CodeAnalysisRuleSet>
    <CodeAnalysisRules Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
    <CodeAnalysisRuleAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|x64'" />
  </PropertyGroup>
  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
    <CustomBuildStep>
      <Message>Build PC/pythonnt_rc(_d).h</Message>
      <Command>cd $(SolutionDir)
make_versioninfo.exe &gt; ..\PC\pythonnt_rc.h
</Command>
      <Outputs>$(SolutionDir)..\PC\pythonnt_rc.h;%(Outputs)</Outputs>
      <Inputs>$(SolutionDir)make_versioninfo.exe</Inputs>
    </CustomBuildStep>
    <ClCompile>
      <Optimization>MaxSpeed</Optimization>
      <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
      <IntrinsicFunctions>true</IntrinsicFunctions>
      <AdditionalIncludeDirectories>%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
      <PreprocessorDefinitions>_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
      <StringPooling>true</StringPooling>
      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
      <FunctionLevelLinking>true</FunctionLevelLinking>
      <CompileAs>Default</CompileAs>
    </ClCompile>
    <Link>
      <OutputFile>$(SolutionDir)make_versioninfo.exe</OutputFile>
      <SubSystem>Console</SubSystem>
      <BaseAddress>0x1d000000</BaseAddress>
    </Link>
    <PostBuildEvent />
    <PostBuildEvent>
      <Command>
      </Command>
    </PostBuildEvent>
  </ItemDefinitionGroup>
  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
    <CustomBuildStep>
      <Message>Build PC/pythonnt_rc(_d).h</Message>
      <Command>cd $(SolutionDir)
make_versioninfo.exe &gt; ..\PC\pythonnt_rc.h
</Command>
      <Outputs>$(SolutionDir)..\PC\pythonnt_rc.h;%(Outputs)</Outputs>
    </CustomBuildStep>
    <ClCompile>
      <Optimization>MaxSpeed</Optimization>
      <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
      <IntrinsicFunctions>true</IntrinsicFunctions>
      <PreprocessorDefinitions>_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
    </ClCompile>
    <Link>
      <OutputFile>$(SolutionDir)make_versioninfo.exe</OutputFile>
    </Link>
    <PostBuildEvent>
      <Command>cd $(SolutionDir)
make_versioninfo.exe &gt; ..\PC\python_nt.h
</Command>
    </PostBuildEvent>
  </ItemDefinitionGroup>
  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
    <CustomBuildStep>
      <Message>Build PC/pythonnt_rc(_d).h</Message>
      <Command>cd $(SolutionDir)
make_versioninfo_d.exe &gt; ..\PC\pythonnt_rc_d.h
</Command>
      <Outputs>$(SolutionDir)..\PC\pythonnt_rc_d.h;%(Outputs)</Outputs>
      <Inputs>$(SolutionDir)make_versioninfo_d.exe</Inputs>
    </CustomBuildStep>
    <ClCompile>
      <Optimization>Disabled</Optimization>
      <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
      <IntrinsicFunctions>false</IntrinsicFunctions>
      <AdditionalIncludeDirectories>%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
      <PreprocessorDefinitions>_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
      <StringPooling>true</StringPooling>
      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
      <FunctionLevelLinking>true</FunctionLevelLinking>
      <CompileAs>Default</CompileAs>
    </ClCompile>
    <Link>
      <OutputFile>$(SolutionDir)make_versioninfo_d.exe</OutputFile>
      <SubSystem>Console</SubSystem>
      <BaseAddress>0x1d000000</BaseAddress>
    </Link>
    <PostBuildEvent>
      <Command>
      </Command>
    </PostBuildEvent>
  </ItemDefinitionGroup>
  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
    <CustomBuildStep>
      <Message>Build PC/pythonnt_rc(_d).h</Message>
      <Command>cd $(SolutionDir)
make_versioninfo_d.exe &gt; ..\PC\pythonnt_rc_d.h
</Command>
      <Outputs>$(SolutionDir)..\PC\pythonnt_rc_d.h;%(Outputs)</Outputs>
    </CustomBuildStep>
    <Midl>
      <TargetEnvironment>X64</TargetEnvironment>
    </Midl>
    <ClCompile>
      <Optimization>Disabled</Optimization>
      <InlineFunctionExpansion>OnlyExplicitInline</InlineFunctionExpansion>
      <IntrinsicFunctions>false</IntrinsicFunctions>
      <PreprocessorDefinitions>_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
    </ClCompile>
    <Link>
      <OutputFile>$(SolutionDir)make_versioninfo_d.exe</OutputFile>
      <TargetMachine>MachineX64</TargetMachine>
    </Link>
    <PostBuildEvent>
      <Command>cd $(SolutionDir)
make_versioninfo_d.exe &gt; ..\PC\python_nt_d.h
</Command>
    </PostBuildEvent>
  </ItemDefinitionGroup>
  <ItemGroup>
    <ClCompile Include="..\PC\make_versioninfo.c" />
  </ItemGroup>
  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
  <ImportGroup Label="ExtensionTargets">
  </ImportGroup>
</Project>

Showing with 0 additions and 0 deletions (0 / 0 diffs computed)

Computing file changes ...