UI tests should be able to test both dev and store-released builds (#567)

Fixes #462

- Add x64 and x86 UI tests to the release builds (internal builds for the store)
- UI tests now accept the target app ID as a parameter in the runsettings file

build/pipelines/azure-pipelines.ci.yaml

@@ -36,6 +36,7 @@ jobs:
 - template: ./templates/run-ui-tests.yaml
   parameters:
     platform: x64
+    runsettingsFileName: CalculatorUITests.ci.runsettings
 
 - template: ./templates/run-unit-tests.yaml
   parameters:

build/pipelines/azure-pipelines.release.yaml

@@ -35,6 +35,16 @@ jobs:
     platform: ARM64
     condition: not(eq(variables['Build.Reason'], 'PullRequest'))
 
+- template: ./templates/run-ui-tests.yaml
+  parameters:
+    platform: x64
+    runsettingsFileName: CalculatorUITests.release.runsettings
+
+- template: ./templates/run-ui-tests.yaml
+  parameters:
+    platform: x86
+    runsettingsFileName: CalculatorUITests.release.runsettings
+
 - template: ./templates/run-unit-tests.yaml
   parameters:
     platform: x64

build/pipelines/templates/run-ui-tests.yaml

@@ -2,6 +2,7 @@
 
 parameters:
   platform: ''
+  runsettingsFileName: ''
 
 jobs:
 - job: UITests${{ parameters.platform }}
@@ -44,6 +45,6 @@ jobs:
     inputs:
       testAssemblyVer2: $(Build.ArtifactStagingDirectory)/drop/Release/${{ parameters.platform }}/publish/CalculatorUITests.dll
       vsTestVersion: 16.0
-      runSettingsFile: $(Build.ArtifactStagingDirectory)/drop/Release/${{ parameters.platform }}/publish/CalculatorUITests.runsettings
+      runSettingsFile: $(Build.ArtifactStagingDirectory)/drop/Release/${{ parameters.platform }}/publish/${{ parameters.runsettingsFileName }}
       platform: ${{ parameters.platform }}
       configuration: Release

src/CalculatorUITestFramework/WinAppDriver.cs

@@ -12,7 +12,7 @@ namespace CalculatorUITestFramework
     public sealed class WinAppDriver
     {
         private WindowsDriverLocalService windowsDriverService = null;
-        private const string calculatorAppId = "Microsoft.WindowsCalculator.Dev_8wekyb3d8bbwe!App";
+        private const string defaultAppId = "Microsoft.WindowsCalculator.Dev_8wekyb3d8bbwe!App";
         private static WinAppDriver instance = null;
         public static WinAppDriver Instance
         {
@@ -55,7 +55,16 @@ public void SetupCalculatorSession(TestContext context)
                 // Create a new  WinAppDriver session to bring up an instance of the Calculator application
                 // Note: Multiple calculator windows (instances) share the same process Id
                 var options = new AppiumOptions();
-                options.AddAdditionalCapability("app", calculatorAppId);
+
+                if (context.Properties.TryGetValue("AppId", out object appId))
+                {
+                    options.AddAdditionalCapability("app", (string)appId);
+                }
+                else
+                {
+                    options.AddAdditionalCapability("app", defaultAppId);
+                }
+
                 options.AddAdditionalCapability("deviceName", "WindowsPC");
                 this.CalculatorSession = new WindowsDriver<WindowsElement>(this.windowsDriverService.ServiceUrl, options);
                 this.CalculatorSession.Manage().Timeouts().ImplicitWait = TimeSpan.FromSeconds(10);

src/CalculatorUITests/CalculatorUITests.ci.runsettings

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="utf-8"?>
+<RunSettings>
+    <DataCollectionRunSettings>
+        <DataCollectors>
+            <DataCollector uri="datacollector://microsoft/VideoRecorder/1.0" assemblyQualifiedName="Microsoft.VisualStudio.TestTools.DataCollection.VideoRecorder.VideoRecorderDataCollector, Microsoft.VisualStudio.TestTools.DataCollection.VideoRecorder, Version=15.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a" friendlyName="Screen and Voice Recorder">
+            </DataCollector>
+        </DataCollectors>
+    </DataCollectionRunSettings>
+    <TestRunParameters>
+        <Parameter Name="AppId" Value="Microsoft.WindowsCalculator.Dev_8wekyb3d8bbwe!App" />
+    </TestRunParameters>
+</RunSettings>

src/CalculatorUITests/CalculatorUITests.csproj

@@ -18,7 +18,10 @@
   </ItemGroup>
 
   <ItemGroup>
-    <None Update="CalculatorUITests.runsettings">
+    <None Update="CalculatorUITests.release.runsettings">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Update="CalculatorUITests.ci.runsettings">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
   </ItemGroup>

src/CalculatorUITests/CalculatorUITests.release.runsettings

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="utf-8"?>
+<RunSettings>
+    <DataCollectionRunSettings>
+        <DataCollectors>
+            <DataCollector uri="datacollector://microsoft/VideoRecorder/1.0" assemblyQualifiedName="Microsoft.VisualStudio.TestTools.DataCollection.VideoRecorder.VideoRecorderDataCollector, Microsoft.VisualStudio.TestTools.DataCollection.VideoRecorder, Version=15.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a" friendlyName="Screen and Voice Recorder">
+            </DataCollector>
+        </DataCollectors>
+    </DataCollectionRunSettings>
+    <TestRunParameters>
+        <Parameter Name="AppId" Value="Microsoft.WindowsCalculator_8wekyb3d8bbwe!App" />
+    </TestRunParameters>
+</RunSettings>

src/CalculatorUITests/CalculatorUITests.runsettings

@@ -1,9 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<RunSettings>
-  <DataCollectionRunSettings>
-    <DataCollectors>
-      <DataCollector uri="datacollector://microsoft/VideoRecorder/1.0" assemblyQualifiedName="Microsoft.VisualStudio.TestTools.DataCollection.VideoRecorder.VideoRecorderDataCollector, Microsoft.VisualStudio.TestTools.DataCollection.VideoRecorder, Version=15.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a" friendlyName="Screen and Voice Recorder">
-      </DataCollector>
-    </DataCollectors>
-  </DataCollectionRunSettings>
-</RunSettings>