Implement Delivery Optimization (#909)

## Change This change adds code to leverage the Delivery Optimization (DO) service to download packages. Only packages are downloaded through DO at this time, as other things that we download do not fit in with the design of DO. Currently one must still opt-in to using DO via settings, which I used rather than an experimental feature because it makes sense to allow the control long term. Eventually, we will make DO the default value here. ## Settings Two settings are added: 1. Control over which downloader is used for packages (`.network.downloader`) 2. A timeout for controlling how long to wait before DO indicates any progress has been made (`.network.doProgressTimeoutInSeconds`)
microsoft · Apr 30, 2021 · 9399b6a · 9399b6a
1 parent cf6f8e5
commit 9399b6a
Show file tree

Hide file tree

Showing 21 changed files with 1,175 additions and 31 deletions.
diff --git a/.github/actions/spelling/allow.txt b/.github/actions/spelling/allow.txt
@@ -3,7 +3,7 @@ ACCESSDENIED
 addmanifest
 addstore
 admins
-Alloc
+alloc
 api
 appdata
 appinst
@@ -21,6 +21,8 @@ argv
 ARRAYSIZE
 aspirational
 aspnet
+authn
+authz
 autocomplete
 auxdata
 azureedge
@@ -30,7 +32,7 @@ Bitmask
 blog
 Blog
 boolalpha
-BSTR
+bstr
 bugfix
 BUILDNUMBER
 bytearray
@@ -104,6 +106,7 @@ ensureandinsert
 ensurepathexists
 ENU
 enum
+EOAC
 errorlevel
 errstr
 esrp
@@ -166,6 +169,7 @@ icu
 IDisposable
 IDX
 IEnumerable
+IFACEMETHOD
 ifdef
 ifndef
 ifstream
@@ -503,7 +507,7 @@ wil
 WINAPI
 WINEVENT
 winget
-WININET
+wininet
 winmeta
 winres
 winrt

diff --git a/.github/actions/spelling/excludes.txt b/.github/actions/spelling/excludes.txt
@@ -22,3 +22,4 @@
 ^src/AppInstallerCLITests/TestData/InputNames.txt$
 ^src/AppInstallerCLITests/TestData/InputPublishers.txt$
 ^src/AppInstallerCLITests/TestData/NormalizationInitialIds.txt$
+^src/AppInstallerCommonCore/external/do.h$
diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt
@@ -242,6 +242,7 @@ UChars
 uec
 uild
 uintptr
+ul
 Uninitialize
 uninstallation
 uninstaller

diff --git a/doc/Settings.md b/doc/Settings.md
@@ -83,6 +83,22 @@ See [details on telemetry](../README.md#datatelemetry), and our [primary privacy
 
 If set to true, the `telemetry.disable` setting will prevent any event from being written by the program.
 
+## Network
+
+The `network` settings influence how winget uses the network to retrieve packages and metadata.
+
+### Downloader
+
+The `downloader` setting controls which code is used when downloading packages. The default is `default`, which may be any of the options based on our determination.
+`wininet` uses the [WinINet](https://docs.microsoft.com/en-us/windows/win32/wininet/about-wininet) APIs, while `do` uses the
+[Delivery Optimization](https://support.microsoft.com/en-us/windows/delivery-optimization-in-windows-10-0656e53c-15f2-90de-a87a-a2172c94cf6d) service.
+
+```json
+   "network": {
+       "downloader": "do"
+   }
+```
+
 ## Experimental Features
 
 To allow work to be done and distributed to early adopters for feedback, settings can be used to enable "experimental" features. 

diff --git a/schemas/JSON/settings/settings.schema.0.2.json b/schemas/JSON/settings/settings.schema.0.2.json
@@ -40,7 +40,8 @@
           "description": "The scope of a package install",
           "type": "string",
           "enum": [
-              "user", "machine"
+            "user",
+            "machine"
           ],
           "default": "user"
         }
@@ -50,8 +51,8 @@
       "description": "Install settings",
       "type": "object",
       "properties": {
-        "preferences": { "$ref": "#/definitions/InstallPrefReq"},
-        "requirements": { "$ref": "#/definitions/InstallPrefReq"}
+        "preferences": { "$ref": "#/definitions/InstallPrefReq" },
+        "requirements": { "$ref": "#/definitions/InstallPrefReq" }
       }
     },
     "Telemetry": {
@@ -65,6 +66,29 @@
         }
       }
     },
+    "Network": {
+      "description": "Network settings",
+      "type": "object",
+      "properties": {
+        "downloader": {
+          "description": "Control which download code is used for packages",
+          "type": "string",
+          "enum": [
+            "default",
+            "wininet",
+            "do"
+          ],
+          "default": "default"
+        },
+        "doProgressTimeoutInSeconds": {
+          "description": "Number of seconds to wait without progress before fallback",
+          "type": "integer",
+          "default": 20,
+          "minimum": 1,
+          "maximum": 600
+        }
+      }
+    },
     "Experimental": {
       "description": "Experimental Features",
       "type": "object",
@@ -99,11 +123,6 @@
           "type": "boolean",
           "default": false
         },
-        "import": {
-          "description": "Enable the import command while it is in development",
-          "type": "boolean",
-          "default": false
-        },
         "restSource": {
           "description": "Enable the rest source support while it is in development",
           "type": "boolean",
@@ -121,25 +140,31 @@
     },
     {
       "properties": {
-        "source": { "$ref": "#/definitions/Source"}
+        "source": { "$ref": "#/definitions/Source" }
+      },
+      "additionalItems": true
+    },
+    {
+      "properties": {
+        "installBehavior": { "$ref": "#/definitions/InstallBehavior" }
       },
       "additionalItems": true
     },
     {
       "properties": {
-        "installBehavior": { "$ref": "#/definitions/InstallBehavior"}
+        "telemetry": { "$ref": "#/definitions/Telemetry" }
       },
       "additionalItems": true
     },
     {
       "properties": {
-        "telemetry": { "$ref": "#/definitions/Telemetry"}
+        "network": { "$ref": "#/definitions/Network" }
       },
       "additionalItems": true
     },
     {
       "properties": {
-        "experimentalFeatures": { "$ref": "#/definitions/Experimental"}
+        "experimentalFeatures": { "$ref": "#/definitions/Experimental" }
       },
       "additionalItems": true
     }

diff --git a/src/AppInstallerCLICore/Workflows/InstallFlow.cpp b/src/AppInstallerCLICore/Workflows/InstallFlow.cpp
@@ -105,6 +105,9 @@ namespace AppInstaller::CLI::Workflow
         std::filesystem::path tempInstallerPath = Runtime::GetPathTo(Runtime::PathName::Temp);
         tempInstallerPath /= Utility::ConvertToUTF16(manifest.Id + '.' + manifest.Version);
 
+        // Use the SHA256 hash of the installer as the identifier for the download
+        std::string hashString = SHA256::ConvertToString(installer.Sha256);
+
         AICLI_LOG(CLI, Info, << "Generated temp download path: " << tempInstallerPath);
 
         context.Reporter.Info() << "Downloading " << Execution::UrlEmphasis << installer.Url << std::endl;
@@ -120,8 +123,10 @@ namespace AppInstaller::CLI::Workflow
                 hash = context.Reporter.ExecuteWithProgress(std::bind(Utility::Download,
                     installer.Url,
                     tempInstallerPath,
+                    Utility::DownloadType::Installer,
                     std::placeholders::_1,
-                    true));
+                    true,
+                    hashString));
 
                 success = true;
             }

diff --git a/src/AppInstallerCLITests/Downloader.cpp b/src/AppInstallerCLITests/Downloader.cpp
@@ -16,7 +16,7 @@ TEST_CASE("DownloadValidFileAndVerifyHash", "[Downloader]")
 
     // Todo: point to files from our repo when the repo goes public
     ProgressCallback callback;
-    auto result = Download("https://raw.githubusercontent.com/microsoft/msix-packaging/master/LICENSE", tempFile.GetPath(), callback, true);
+    auto result = Download("https://raw.githubusercontent.com/microsoft/msix-packaging/master/LICENSE", tempFile.GetPath(), DownloadType::Manifest, callback, true);
 
     REQUIRE(result.has_value());
     auto resultHash = result.value();
@@ -49,7 +49,7 @@ TEST_CASE("DownloadValidFileAndCancel", "[Downloader]")
     std::optional<std::vector<BYTE>> waitResult;
     std::thread waitThread([&]
         {
-            waitResult = Download("https://aka.ms/win32-x64-user-stable", tempFile.GetPath(), callback, true);
+            waitResult = Download("https://aka.ms/win32-x64-user-stable", tempFile.GetPath(), DownloadType::Installer, callback, true);
         });
 
     callback.Cancel();
@@ -66,5 +66,5 @@ TEST_CASE("DownloadInvalidUrl", "[Downloader]")
 
     ProgressCallback callback;
 
-    REQUIRE_THROWS_HR(Download("blargle-flargle-fluff", tempFile.GetPath(), callback, true), WININET_E_UNRECOGNIZED_SCHEME);
+    REQUIRE_THROWS_HR(Download("blargle-flargle-fluff", tempFile.GetPath(), DownloadType::Installer, callback, true), WININET_E_UNRECOGNIZED_SCHEME);
 }
diff --git a/src/AppInstallerCommonCore/AppInstallerCommonCore.vcxproj b/src/AppInstallerCommonCore/AppInstallerCommonCore.vcxproj
@@ -260,6 +260,7 @@
     </Link>
   </ItemDefinitionGroup>
   <ItemGroup>
+    <ClInclude Include="DODownloader.h" />
     <ClInclude Include="Public\winget\GroupPolicy.h" />
     <ClInclude Include="HttpStream\HttpClientWrapper.h" />
     <ClInclude Include="HttpStream\HttpLocalCache.h" />
@@ -307,6 +308,7 @@
     <ClInclude Include="YamlWrapper.h" />
   </ItemGroup>
   <ItemGroup>
+    <ClCompile Include="DODownloader.cpp" />
     <ClCompile Include="GroupPolicy.cpp">
       <ExcludedFromBuild Condition="'$(Configuration)'=='Fuzzing'">true</ExcludedFromBuild>
     </ClCompile>

diff --git a/src/AppInstallerCommonCore/AppInstallerCommonCore.vcxproj.filters b/src/AppInstallerCommonCore/AppInstallerCommonCore.vcxproj.filters
@@ -165,6 +165,9 @@
     <ClInclude Include="Public\winget\Resources.h">
       <Filter>Public\winget</Filter>
     </ClInclude>
+    <ClInclude Include="DODownloader.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="pch.cpp">
@@ -278,6 +281,9 @@
     <ClCompile Include="GroupPolicy.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="DODownloader.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <None Include="PropertySheet.props" />
-Original file line number
+Diff line change
@@ Expand Up / @@ -242,6 +242,7 @@ UChars @@
     uec
     uild
     uintptr
+    ul
     Uninitialize
     uninstallation
     uninstaller
@@ Expand Down @@