Move UI tests to Windows 11 (#1866)

Calculator's UI tests currently run on Server 2022 VM images. However, the production app is primarily developed for Windows 11. This change moves the UI test jobs to run on Windows 11, so we can run our tests in an environment that's closer to what real users will experience.

build/pipelines/azure-pipelines.release.yaml

@@ -36,6 +36,16 @@ jobs:
     useReleaseAppxmanifest: true
     condition: not(eq(variables['Build.Reason'], 'PullRequest'))
 
+- template: ./templates/run-ui-tests.yaml
+  parameters:
+    platform: x64
+    runsettingsFileName: CalculatorUITests.release.runsettings
+
+- template: ./templates/run-ui-tests.yaml
+  parameters:
+    platform: x86
+    runsettingsFileName: CalculatorUITests.release.runsettings
+
 - template: ./templates/run-unit-tests.yaml
   parameters:
     platform: x64

build/pipelines/templates/run-ui-tests.yaml

@@ -12,17 +12,26 @@ jobs:
   condition: succeeded()
   pool:
     ${{ if eq(parameters.isOSSBuild, true) }}:
-      name: EssentialExperiencesOpenSource-windows-2022
+      name: EssentialExperiencesOpenSource-Win11
     ${{ if eq(parameters.isOSSBuild, false) }}:
-      name: EssentialExperiences-windows-2022
+      name: EssentialExperiences-Win11
   variables:
     skipComponentGovernanceDetection: true
   steps:
-  - checkout: none
+  - checkout: self
+    fetchDepth: 1
 
-  - powershell: Set-DisplayResolution -Width 1920 -Height 1080 -Force	
+  - task: PowerShell@2
+    displayName: Turn off animation effects
+    inputs:
+      filePath: $(Build.SourcesDirectory)\build\scripts\TurnOffAnimationEffects.ps1
+
+  - task: ScreenResolutionUtility@1
     displayName: Set resolution to 1920x1080
-    continueOnError: true	
+    inputs:
+      displaySettings: 'specific'
+      width: 1920
+      height: 1080
 
   - task: DownloadBuildArtifacts@0
     displayName: Download MsixBundle and CalculatorUITests

build/scripts/TurnOffAnimationEffects.ps1

@@ -0,0 +1,22 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+<#
+.SYNOPSIS
+    Disables animations on the system. Equivalent to turning off the "Animation effects" setting in the Windows Settings app.
+#>
+
+Add-Type -AssemblyName System.Runtime.WindowsRuntime
+$asTaskGeneric = ([System.WindowsRuntimeSystemExtensions].GetMethods() | ? { $_.Name -eq 'AsTask' -and $_.GetParameters().Count -eq 1 -and $_.GetParameters()[0].ParameterType.Name -eq 'IAsyncOperation`1' })[0]
+
+Function WaitForAsyncAction($WinRtTask, $ResultType) {
+    $asTask = $asTaskGeneric.MakeGenericMethod($ResultType)
+    $task = $asTask.Invoke($null, @($WinRtTask))
+    $task.GetAwaiter().GetResult()
+}
+
+[Windows.UI.ViewManagement.Core.UISettingsController,Windows.UI.ViewManagement.Core,ContentType=WindowsRuntime] | Out-Null
+
+$controller = WaitForAsyncAction ([Windows.UI.ViewManagement.Core.UISettingsController]::RequestDefaultAsync())  ([Windows.UI.ViewManagement.Core.UISettingsController])
+
+$controller.SetAnimationsEnabled($false)

src/Calculator/Package.appxmanifest

@@ -8,7 +8,7 @@
         <Logo>Assets\CalculatorStoreLogo.png</Logo>
     </Properties>
     <Dependencies>
-        <TargetDeviceFamily Name="Windows.Universal" MinVersion="10.0.18362.0" MaxVersionTested="10.0.22000.0" />
+        <TargetDeviceFamily Name="Windows.Universal" MinVersion="10.0.22000.0" MaxVersionTested="10.0.22000.0" />
     </Dependencies>
     <Resources>
         <Resource Language="x-generate" />

src/CalculatorUITestFramework/CalculatorUITestFramework.csproj

@@ -1,6 +1,6 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
-    <TargetFramework>netstandard2.0</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
   </PropertyGroup>
   <ItemGroup>
     <PackageReference Include="Appium.WebDriver" Version="4.3.1" />

src/CalculatorUITestFramework/ProgrammerOperatorsPanel.cs

@@ -172,17 +172,17 @@ public void ResetWordSize()
             else if (source.Contains("dwordButton"))
             {
                 DWordButton.Click();
-                WordButton.Click();
+                ResetWordSize();
-                ByteButton.Click();
             }
             else if (source.Contains("wordButton"))
             {
                 WordButton.Click();
-                ByteButton.Click();
+                ResetWordSize();
             }
             else if (source.Contains("byteButton"))
             {
                 ByteButton.Click();
+                ResetWordSize();
             }
             else
             {

src/CalculatorUITestFramework/StandardAoTCalculatorPage.cs

@@ -95,40 +95,29 @@ public string GetAoTToolTipText()
             return ToolTip.Text;
         }
 
-        ///// <summary>
+        /// <summary>
-        ///// Checks in AoT (Keep on top) button is present
+        /// Checks in AoT (Keep on top) button is present
-        ///// </summary>
+        /// </summary>
-        public string GetAoTPresence()
+        public bool IsKeepOnTopButtonPresent()
         {
-            bool AoTPresent;
             string source = this.session.PageSource;
-            if (source.Contains("Keep on top"))
+            return source.Contains("Keep on top");
-            {
-                AoTPresent = true;
-            }
-            else
-            {
-                AoTPresent = false;
-            }
-            return AoTPresent.ToString();
         }
 
-        ///// <summary>
+        /// <summary>
-        ///// Checks Standard calculator to see if it's in AoT (Keep on top)
+        /// Checks Standard calculator to see if it's in AoT (Keep on top)
-        ///// </summary>
+        /// </summary>
-        public string AoTModeCheck()
+        public bool IsInAlwaysOnTopMode()
         {
-            bool InAoTMode;
             string source = this.session.PageSource;
             if ((source.Contains("Keep on top")) && (source.Contains("Header")))
             {
-                InAoTMode = false;
+                return false;
             }
             else
             {
-                InAoTMode = true;
+                return true;
             }
-            return InAoTMode.ToString();
         }
 
         /// <summary>
@@ -143,11 +132,11 @@ public void ResizeAoTWindowToDisplayInvertButton()
         }
 
         /// <summary>
-        /// Increases the size of the window until History label for the History panel is visible
+        /// Increases the height of the window until invert button is visible
         /// </summary>
         private void GrowWindowToShowInvertButton(int height)
         {
-            if (height > 1000)
+            if (height > 700)
             {
                 throw new NotFoundException("Could not find the Invert Button");
             }
@@ -158,45 +147,7 @@ private void GrowWindowToShowInvertButton(int height)
                 WinAppDriver.Instance.CalculatorSession.Manage().Window.Size = new Size(width, height);
                 //give window time to render new size
                 System.Threading.Thread.Sleep(10);
-                GrowWindowToShowInvertButton(width + 100);
+                GrowWindowToShowInvertButton(height + 100);
-            }
-        }
-        /// <summary>
-        /// If the Invert button is not displayed, resize the window
-        /// Two attempts are made, the the button is not found a "not found" exception is thrown
-        /// </summary>
-        public void ResizeAoTWindowToDiplayInvertButton()
-        {
-            Point newWindowPostion = new Point(8, 8);
-            WinAppDriver.Instance.CalculatorSession.Manage().Window.Position = newWindowPostion;
-            string source0 = this.session.PageSource;
-            if (source0.Contains("invertButton"))
-            {
-                return;
-            }
-            else
-            {
-                Size newWindowSize = new Size(502, 502);
-                WinAppDriver.Instance.CalculatorSession.Manage().Window.Size = newWindowSize;
-                string source1 = this.session.PageSource;
-                if (source1.Contains("invertButton"))
-                {
-                    return;
-                }
-                else
-                {
-                    Size newWindowSize2 = new Size(750, 750);
-                    WinAppDriver.Instance.CalculatorSession.Manage().Window.Size = newWindowSize2;
-                }
-                string source2 = this.session.PageSource;
-                if (source2.Contains("invertButton"))
-                {
-                    return;
-                }
-                else
-                {
-                    throw new NotFoundException("Could not find the Invert Button");
-                }
             }
         }
     }

src/CalculatorUITestFramework/UnitConverterPage.cs

@@ -68,11 +68,9 @@ public void EnsureSameUnitsAreSelected()
         {
             CalculatorApp.ClickOnWindow();
             UnitConverterOperators.Units1.SendKeys(OpenQA.Selenium.Keys.Home);
-            UnitConverterOperators.Units1.SendKeys(OpenQA.Selenium.Keys.Enter);
 
             CalculatorApp.ClickOnWindow();
             UnitConverterOperators.Units2.SendKeys(OpenQA.Selenium.Keys.Home);
-            UnitConverterOperators.Units2.SendKeys(OpenQA.Selenium.Keys.Enter);
 
             CalculatorApp.ClickOnWindow();
         }

src/CalculatorUITests/CalculatorUITests.csproj

@@ -1,6 +1,6 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
-    <TargetFramework>netcoreapp3.1</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
     <IsPackable>false</IsPackable>
   </PropertyGroup>
   <ItemGroup>

src/CalculatorUITests/StandardModeFunctionalTests.cs

@@ -181,10 +181,10 @@ public void MouseInput_HistoryButtons()
             Assert.IsTrue(historyFlyoutItems[1].GetExpression().Equals("4 × 5=", StringComparison.InvariantCultureIgnoreCase)); //verifies History button
             page.HistoryPanel.ResizeWindowToDisplayHistoryLabel();
             var historyItems = page.HistoryPanel.GetAllHistoryListViewItems();
-            Assert.IsTrue(historyFlyoutItems[0].GetValue().Equals("3.333333333333333", StringComparison.InvariantCultureIgnoreCase)); //verifies History button
+            Assert.IsTrue(historyItems[0].GetValue().Equals("3.333333333333333", StringComparison.InvariantCultureIgnoreCase)); //verifies History button
-            Assert.IsTrue(historyFlyoutItems[0].GetExpression().Equals("20 ÷ 6=", StringComparison.InvariantCultureIgnoreCase)); //verifies History button
+            Assert.IsTrue(historyItems[0].GetExpression().Equals("20 ÷ 6=", StringComparison.InvariantCultureIgnoreCase)); //verifies History button
-            Assert.IsTrue(historyFlyoutItems[1].GetValue().Equals("20", StringComparison.InvariantCultureIgnoreCase)); //verifies History button
+            Assert.IsTrue(historyItems[1].GetValue().Equals("20", StringComparison.InvariantCultureIgnoreCase)); //verifies History button
-            Assert.IsTrue(historyFlyoutItems[1].GetExpression().Equals("4 × 5=", StringComparison.InvariantCultureIgnoreCase)); //verifies History button
+            Assert.IsTrue(historyItems[1].GetExpression().Equals("4 × 5=", StringComparison.InvariantCultureIgnoreCase)); //verifies History button
             page.HistoryPanel.ClearHistoryButton.Click();
             Assert.IsNotNull(WinAppDriver.Instance.CalculatorSession.FindElementByAccessibilityId("HistoryEmpty")); //verifies the History panel's clear history button
         }
@@ -666,7 +666,6 @@ public void AoT_EnterExitKeepOnTop()
         {
             page.StandardAoTCalculatorPage.NavigateToStandardAoTMode();
             page.StandardAoTCalculatorPage.NavigateToStandardMode();
-
         }
 
         [TestMethod]
@@ -724,26 +723,22 @@ public void AoT_ButtonOnlyInStandard()
         {
             page.NavigationMenu.ChangeCalculatorMode(CalculatorMode.ScientificCalculator);
             Assert.AreEqual("Scientific", CalculatorApp.GetCalculatorHeaderText());
-            page.StandardAoTCalculatorPage.GetAoTPresence();
+            Assert.IsFalse(page.StandardAoTCalculatorPage.IsKeepOnTopButtonPresent());
-            Assert.AreEqual("False", page.StandardAoTCalculatorPage.GetAoTPresence());
 
             CalculatorApp.EnsureCalculatorHasFocus();
             page.NavigationMenu.ChangeCalculatorMode(CalculatorMode.ProgrammerCalculator);
             Assert.AreEqual("Programmer", CalculatorApp.GetCalculatorHeaderText());
-            page.StandardAoTCalculatorPage.GetAoTPresence();
+            Assert.IsFalse(page.StandardAoTCalculatorPage.IsKeepOnTopButtonPresent());
-            Assert.AreEqual("False", page.StandardAoTCalculatorPage.GetAoTPresence());
 
             CalculatorApp.EnsureCalculatorHasFocus();
             page.NavigationMenu.ChangeCalculatorMode(CalculatorMode.DateCalculator);
             Assert.AreEqual("Date calculation", CalculatorApp.GetCalculatorHeaderText());
-            page.StandardAoTCalculatorPage.GetAoTPresence();
+            Assert.IsFalse(page.StandardAoTCalculatorPage.IsKeepOnTopButtonPresent());
-            Assert.AreEqual("False", page.StandardAoTCalculatorPage.GetAoTPresence());
 
             CalculatorApp.EnsureCalculatorHasFocus();
             page.NavigationMenu.ChangeCalculatorMode(CalculatorMode.StandardCalculator);
             Assert.AreEqual("Standard", CalculatorApp.GetCalculatorHeaderText());
-            page.StandardAoTCalculatorPage.GetAoTPresence();
+            Assert.IsTrue(page.StandardAoTCalculatorPage.IsKeepOnTopButtonPresent());
-            Assert.AreEqual("True", page.StandardAoTCalculatorPage.GetAoTPresence());
         }
 
         [TestMethod]
@@ -751,12 +746,11 @@ public void AoT_ButtonOnlyInStandard()
         public void AoT_ErrorMessage_ResultUndefined()
         {
             page.StandardAoTCalculatorPage.NavigateToStandardAoTMode();
-            page.StandardAoTCalculatorPage.ResizeAoTWindowToDisplayInvertButton();
+            Assert.IsTrue(page.StandardAoTCalculatorPage.IsInAlwaysOnTopMode());
+
             page.StandardOperators.DivideButton.Click();
             page.StandardOperators.NumberPad.Num0Button.Click();
             page.StandardOperators.EqualButton.Click();
-            page.StandardAoTCalculatorPage.AoTModeCheck();
-            Assert.AreEqual("True", page.StandardAoTCalculatorPage.AoTModeCheck());
             Assert.AreEqual("Result is undefined", page.CalculatorResults.GetAoTCalculatorResultText());
         }
 
@@ -765,11 +759,11 @@ public void AoT_ErrorMessage_ResultUndefined()
         public void AoT_ErrorMessage_CannotDivideByZero()
         {
             page.StandardAoTCalculatorPage.NavigateToStandardAoTMode();
+            Assert.IsTrue(page.StandardAoTCalculatorPage.IsInAlwaysOnTopMode());
             page.StandardAoTCalculatorPage.ResizeAoTWindowToDisplayInvertButton();
+
             page.StandardOperators.ClearButton.Click();
             page.StandardOperators.InvertButton.Click();
-            page.StandardAoTCalculatorPage.AoTModeCheck();
-            Assert.AreEqual("True", page.StandardAoTCalculatorPage.AoTModeCheck());
             Assert.AreEqual("Cannot divide by zero", page.CalculatorResults.GetAoTCalculatorResultText());
         }
 
@@ -778,16 +772,16 @@ public void AoT_ErrorMessage_CannotDivideByZero()
         public void AoT_ErrorMessage_MessageRetentionUponExitingAoT()
         {
             page.StandardAoTCalculatorPage.NavigateToStandardAoTMode();
-            page.StandardAoTCalculatorPage.ResizeAoTWindowToDisplayInvertButton();
+            Assert.IsTrue(page.StandardAoTCalculatorPage.IsInAlwaysOnTopMode());
-            page.StandardOperators.ClearButton.Click();
+
-            page.StandardOperators.InvertButton.Click();
+            page.StandardOperators.DivideButton.Click();
-            page.StandardAoTCalculatorPage.AoTModeCheck();
+            page.StandardOperators.NumberPad.Num0Button.Click();
-            Assert.AreEqual("True", page.StandardAoTCalculatorPage.AoTModeCheck());
+            page.StandardOperators.EqualButton.Click();
-            Assert.AreEqual("Cannot divide by zero", page.CalculatorResults.GetAoTCalculatorResultText());
+            Assert.AreEqual("Result is undefined", page.CalculatorResults.GetAoTCalculatorResultText());
+
             page.StandardAoTCalculatorPage.NavigateToStandardMode();
-            page.StandardAoTCalculatorPage.AoTModeCheck();
+            Assert.IsFalse(page.StandardAoTCalculatorPage.IsInAlwaysOnTopMode());
-            Assert.AreEqual("False", page.StandardAoTCalculatorPage.AoTModeCheck());
+            Assert.AreEqual("Result is undefined", page.CalculatorResults.GetCalculatorResultText());
-            Assert.AreEqual("Cannot divide by zero", page.CalculatorResults.GetCalculatorResultText());
         }
 
         #endregion