All of lore.kernel.org
 help / color / mirror / Atom feed
* [KVM-AUTOTEST PATCH v2 1/5] [RFC] KVM test: DTM automation program for WHQL tests
@ 2010-07-22 10:14 Michael Goldish
  2010-07-22 10:14 ` [KVM-AUTOTEST PATCH v2 2/5] [RFC] KVM test: DTM machine deletion tool " Michael Goldish
  0 siblings, 1 reply; 8+ messages in thread
From: Michael Goldish @ 2010-07-22 10:14 UTC (permalink / raw)
  To: autotest, kvm

This C# program should run on a DTM/WHQL server.  It's used by the
whql_submission test to schedule and monitor device submission jobs.

Note: the binary is copied to the server at run time, so it doesn't need to be
packaged in winutils.iso.

Changes from v1:
- Take a job filter regex from the user, and run only the jobs whose names
  match the regex.  This is useful for running individual tests instead of
  whole submissions which can take many hours to run.
- When test execution completes set the client machine's state to Unsafe and
  then Reset.
- Instead of looking for the requested test device and failing immediately if
  it isn't found, allow up to 2 minutes for it to show up.
- Add the job ID of each job to the result list printed to stdout.

Signed-off-by: Michael Goldish <mgoldish@redhat.com>
---
 client/tests/kvm/deps/whql_submission_15.cs  |  289 ++++++++++++++++++++++++++
 client/tests/kvm/deps/whql_submission_15.exe |  Bin 0 -> 10240 bytes
 2 files changed, 289 insertions(+), 0 deletions(-)
 create mode 100644 client/tests/kvm/deps/whql_submission_15.cs
 create mode 100644 client/tests/kvm/deps/whql_submission_15.exe

diff --git a/client/tests/kvm/deps/whql_submission_15.cs b/client/tests/kvm/deps/whql_submission_15.cs
new file mode 100644
index 0000000..bf6e136
--- /dev/null
+++ b/client/tests/kvm/deps/whql_submission_15.cs
@@ -0,0 +1,289 @@
+// DTM submission automation program
+// Author: Michael Goldish <mgoldish@redhat.com>
+// Based on sample code by Microsoft.
+
+// Note: this program has only been tested with DTM version 1.5.
+// It might fail to work with other versions, specifically because it uses
+// a few undocumented methods/attributes.
+
+using System;
+using System.Collections.Generic;
+using System.Text.RegularExpressions;
+using Microsoft.DistributedAutomation.DeviceSelection;
+using Microsoft.DistributedAutomation.SqlDataStore;
+
+namespace automate0
+{
+    class AutoJob
+    {
+        static int Main(string[] args)
+        {
+            if (args.Length != 5)
+            {
+                Console.WriteLine("Error: incorrect number of command line arguments");
+                Console.WriteLine("Usage: {0} serverName clientName machinePoolName submissionName timeout",
+                    System.Environment.GetCommandLineArgs()[0]);
+                return 1;
+            }
+            string serverName = args[0];
+            string clientName = args[1];
+            string machinePoolName = args[2];
+            string submissionName = args[3];
+            double timeout = Convert.ToDouble(args[4]);
+
+            try
+            {
+                // Initialize DeviceScript and connect to data store
+                Console.WriteLine("Initializing DeviceScript object");
+                DeviceScript script = new DeviceScript();
+                Console.WriteLine("Connecting to data store");
+
+                script.ConnectToNamedDataStore(serverName);
+
+                // Find client machine
+                IResourcePool rootPool = script.GetResourcePoolByName("$");
+                Console.WriteLine("Looking for client machine '{0}'", clientName);
+                IResource machine = null;
+                while (true)
+                {
+                    try
+                    {
+                        machine = rootPool.GetResourceByName(clientName);
+                    }
+                    catch (Exception e)
+                    {
+                        Console.WriteLine("Warning: " + e.Message);
+                    }
+                    // Make sure the machine is valid
+                    if (machine != null &&
+                        machine.OperatingSystem != null &&
+                        machine.OperatingSystem.Length > 0 &&
+                        machine.ProcessorArchitecture != null &&
+                        machine.ProcessorArchitecture.Length > 0 &&
+                        machine.GetDevices().Length > 0)
+                        break;
+                    System.Threading.Thread.Sleep(1000);
+                }
+                Console.WriteLine("Client machine '{0}' found ({1}, {2})",
+                    clientName, machine.OperatingSystem, machine.ProcessorArchitecture);
+
+                // Create machine pool and add client machine to it
+                // (this must be done because jobs cannot be scheduled for machines in the
+                // default pool)
+                try
+                {
+                    script.CreateResourcePool(machinePoolName, rootPool.ResourcePoolId);
+                }
+                catch (Exception e)
+                {
+                    Console.WriteLine("Warning: " + e.Message);
+                }
+                IResourcePool newPool = script.GetResourcePoolByName(machinePoolName);
+                Console.WriteLine("Moving the client machine to pool '{0}'", machinePoolName);
+                machine.ChangeResourcePool(newPool);
+
+                // Reset client machine
+                if (machine.Status != "Ready")
+                {
+                    Console.WriteLine("Changing the client machine's status to 'Reset'");
+                    while (true)
+                    {
+                        try
+                        {
+                            machine = rootPool.GetResourceByName(clientName);
+                            machine.ChangeResourceStatus("Unsafe");
+                            System.Threading.Thread.Sleep(5000);
+                            machine.ChangeResourceStatus("Reset");
+                            break;
+                        }
+                        catch (Exception e)
+                        {
+                            Console.WriteLine("Warning: " + e.Message);
+                        }
+                        System.Threading.Thread.Sleep(5000);
+                    }
+                    Console.WriteLine("Waiting for client machine to be ready");
+                    while (machine.Status != "Ready")
+                    {
+                        try
+                        {
+                            machine = rootPool.GetResourceByName(clientName);
+                        }
+                        catch (Exception e)
+                        {
+                            Console.WriteLine("Warning: " + e.Message);
+                        }
+                        System.Threading.Thread.Sleep(1000);
+                    }
+                }
+                Console.WriteLine("Client machine is ready");
+
+                // Get requested device regex and look for a matching device
+                Console.WriteLine("Device to test: ");
+                Regex deviceRegex = new Regex(Console.ReadLine(), RegexOptions.IgnoreCase);
+                Console.WriteLine("Looking for device '{0}'", deviceRegex);
+                IDevice device;
+                DateTime endTime = DateTime.Now.AddSeconds(120);
+                while (DateTime.Now < endTime)
+                {
+                    machine = rootPool.GetResourceByName(clientName);
+                    Console.WriteLine("(Client machine has {0} devices)", machine.GetDevices().Length);
+                    foreach (IDevice d in machine.GetDevices())
+                    {
+                        if (deviceRegex.IsMatch(d.FriendlyName))
+                        {
+                            device = d;
+                            goto deviceFound;
+                        }
+                    }
+                    System.Threading.Thread.Sleep(5000);
+                }
+                Console.WriteLine("Error: device '{0}' not found", deviceRegex);
+                return 1;
+
+            deviceFound:
+                Console.WriteLine("Found device '{0}'", device.FriendlyName);
+
+                // Get requested jobs regex
+                Console.WriteLine("Jobs to run: ");
+                Regex jobRegex = new Regex(Console.ReadLine(), RegexOptions.IgnoreCase);
+
+                // Create submission
+                Object[] existingSubmissions = script.GetSubmissionByName(submissionName);
+                if (existingSubmissions.Length > 0)
+                {
+                    Console.WriteLine("Submission '{0}' already exists -- removing it",
+                        submissionName);
+                    script.DeleteSubmission(((ISubmission)existingSubmissions[0]).Id);
+                }
+                Console.WriteLine("Creating submission '{0}'", submissionName);
+                ISubmission submission = script.CreateHardwareSubmission(submissionName,
+                    newPool.ResourcePoolId, device.InstanceId);
+
+                // Get DeviceData objects from the user
+                List<Object> deviceDataList = new List<Object>();
+                while (true)
+                {
+                    ISubmissionDeviceData dd = script.CreateNewSubmissionDeviceData();
+                    Console.WriteLine("DeviceData name: ");
+                    dd.Name = Console.ReadLine();
+                    if (dd.Name.Length == 0)
+                        break;
+                    Console.WriteLine("DeviceData data: ");
+                    dd.Data = Console.ReadLine();
+                    deviceDataList.Add(dd);
+                }
+
+                // Set the submission's DeviceData
+                submission.SetDeviceData(deviceDataList.ToArray());
+
+                // Get descriptors from the user
+                List<Object> descriptorList = new List<Object>();
+                while (true)
+                {
+                    Console.WriteLine("Descriptor path: ");
+                    string descriptorPath = Console.ReadLine();
+                    if (descriptorPath.Length == 0)
+                        break;
+                    descriptorList.Add(script.GetDescriptorByPath(descriptorPath));
+                }
+
+                // Set the submission's descriptors
+                submission.SetLogoDescriptors(descriptorList.ToArray());
+
+                // Create a schedule
+                ISchedule schedule = script.CreateNewSchedule();
+                Console.WriteLine("Scheduling jobs:");
+                int jobCount = 0;
+                foreach (IJob j in submission.GetJobs())
+                {
+                    if (jobRegex.IsMatch(j.Name))
+                     {
+                        Console.WriteLine("  " + j.Name);
+                        schedule.AddDeviceJob(device, j);
+                        jobCount++;
+                    }
+                }
+                if (jobCount == 0)
+                {
+                    Console.WriteLine("Error: no submission jobs match pattern '{0}'", jobRegex);
+                    return 1;
+                }
+                schedule.AddSubmission(submission);
+                schedule.SetResourcePool(newPool);
+                script.RunSchedule(schedule);
+
+                // Wait for jobs to complete
+                Console.WriteLine("Waiting for all jobs to complete (timeout={0})", timeout);
+                endTime = DateTime.Now.AddSeconds(timeout);
+                int numCompleted = 0, numFailed = 0;
+                while (numCompleted < submission.GetResults().Length && DateTime.Now < endTime)
+                {
+                    // Sleep for 30 seconds
+                    System.Threading.Thread.Sleep(30000);
+                    // Count completed submission jobs
+                    numCompleted = 0;
+                    foreach (IResult r in submission.GetResults())
+                        if (r.ResultStatus != "InProgress")
+                            numCompleted++;
+                    // Report results in a Python readable format and count failed schedule jobs
+                    // (submission jobs are a subset of schedule jobs)
+                    Console.WriteLine();
+                    Console.WriteLine("---- [");
+                    numFailed = 0;
+                    foreach (IResult r in schedule.GetResults())
+                    {
+                        Console.WriteLine("  {");
+                        Console.WriteLine("    'id': {0}, 'job': r'''{1}''',", r.Job.Id, r.Job.Name);
+                        Console.WriteLine("    'logs': r'''{0}''',", r.LogLocation);
+                        if (r.ResultStatus != "InProgress")
+                            Console.WriteLine("    'report': r'''{0}''',",
+                                submission.GetSubmissionResultReport(r));
+                        Console.WriteLine("    'status': '{0}',", r.ResultStatus);
+                        Console.WriteLine("    'pass': {0}, 'fail': {1}, 'notrun': {2}, 'notapplicable': {3}",
+                            r.Pass, r.Fail, r.NotRun, r.NotApplicable);
+                        Console.WriteLine("  },");
+                        numFailed += r.Fail;
+                    }
+                    Console.WriteLine("] ----");
+                }
+                Console.WriteLine();
+
+                // Cancel incomplete jobs
+                foreach (IResult r in schedule.GetResults())
+                    if (r.ResultStatus == "InProgress")
+                        r.Cancel();
+
+                // Set the machine's status to Unsafe and then Reset
+                try
+                {
+                    machine = rootPool.GetResourceByName(clientName);
+                    machine.ChangeResourceStatus("Unsafe");
+                    System.Threading.Thread.Sleep(5000);
+                    machine.ChangeResourceStatus("Reset");
+                }
+                catch (Exception e)
+                {
+                    Console.WriteLine("Warning: " + e.Message);
+                }
+
+                // Report failures
+                if (numCompleted < submission.GetResults().Length)
+                    Console.WriteLine("Some jobs did not complete on time.");
+                if (numFailed > 0)
+                    Console.WriteLine("Some jobs failed.");
+
+                if (numFailed > 0 || numCompleted < submission.GetResults().Length)
+                    return 1;
+
+                Console.WriteLine("All jobs completed.");
+                return 0;
+            }
+            catch (Exception e)
+            {
+                Console.WriteLine("Error: " + e.Message);
+                return 1;
+            }
+        }
+    }
+}
diff --git a/client/tests/kvm/deps/whql_submission_15.exe b/client/tests/kvm/deps/whql_submission_15.exe
new file mode 100644
index 0000000000000000000000000000000000000000..4f30aa801c8f200bd96b0de6c10d2b59c2bad268
GIT binary patch
literal 10240
zcmeHMYiu0Xbw0BXa=8-Mnq6x3BBc=@;!-4+q@*O0ELx;UN~TMSG?$X4nyNJ19g(Z-
z&d_&eX)<yPwvJ;5uvG=kkAz0uxab2#(^@GE#7OG2DcZt8ZJ;iSHV;?{S|m-KAO>o<
zNEI~dckayWE=fsMfc`1qA?M7w=braH_s*TW<G*~Jf<zR;efu`iYk2ZCAmP2i3c6!^
zz7nHX>)zV?nlkp*-s$<gtt|*^PUuA~tDC0fXtRbUN~V@Kwc&}3R<v?Py1qWz>ZzU_
zAsSPH^y~+x#(ZnvqkytWi4g4u1rNUd^caAKdjK~B4Ud59H!$fxKWPOw=zIlf;2EMV
z^1t%*nl1~?K<@<O14Q5CB&@$K5p4$Tiw_btuJ5~7TPwRRxguykT*aSuj3o!{S6M0g
zWUQXwXvs?>x|9~S$f6~E8vs&+Tf<#-4Z!=fFbWnJ*;l%Rn|1BOU3Co*ov3O`^g8Di
z7yA~VFcnfzmLO8l#?97#p=(!fh(xWFRRh=ZyV6YpY8FyRZB!VHg@GkkQ_vjUwm*<u
zjRv-@ZVGH)-5l7lS|8ZCnu2sR5fcCDi`giFXd)~QE1n)TTw~RNriR58(5xu;*9a4M
zt<>9)s8wrftxZg?y}2_H5jW7MMl56}nyP2O{@f(Bw(FQ$YsJv$I_M$R1861tTG2%5
zM7wr*h)3UyrV9GCe6n}5_Cw5_R2<FDYvQljzNf^KNg<PPGosd54Zy6e0Q=xjo$FBD
zMu)-?H=L;05}<{M3`eb1qgIkN)mq!|5SP%E+79f<pI~rAftwsqC1|x)ya}qc161uf
zcp;qx_1ix6=Dx%x*NaUXyr@WYeo_3u*WTG%(T^B5yV9FCN|%$e!QbxJ#i5|*tjPIT
zYA3whX72(_)QeAntk%oGQ<xmlL`%G-#jU|#8AvrT<$=yC6<5S-EZ`5y^B4@p%bFag
zL~An9)|paU)wWfK3p=qztyi~LSX-jj9)PYIbxR_4^QhXk?OdWwZBtvX<q!OQB?Zz6
z^?{TI-tOnbt%`fw63y>Ci@b}EhTuU=s+6Y13<z>WszX!NaGkZ6{g110luw@~mcUo%
zdK*^b$yEncqFBxB8bXeWf2!)QLlJNh!B8tRv;ia%Hs*LV(V&K{c1E^}XxJ}?t<t!K
z2_03pS{-=E67Je7((8HlGfxA!I)QZlnVL`=tS%Pl25?Oz#gs24^(MBd+gyEdHK~Tt
zNV9-c@Q+ZCbQSb~2<Lk7NNg92zKUdGhnleZn7S{SXzv_V_o?lx4$4(Y>{L~Cr*(j}
z$?<6ypM#i3s?1(>yBY)WARE|$q1&Oh1GcN}>OKtME9#E!2JsRLsxrYmSSWsiSi(BQ
zLfethk1!Hew_6_tB5pBz=y{Z*TS9LArQ#_y<P}~ru`8KqJbV>vn$}zLrWJ_L`7vln
zbR-k|J3G}5b^oe^dR7umYLj)Cb)T!jxbU1sz3qadyVU)F`_=tw2i$!tb%dErX5yI)
zBXH^{utaAv(RKI(OaQf0?eetl7Ek${;*-GC-4@<YQ45QCRL%gOz*$W71Bf?VwuhL&
zYcDv~`IQhqWu63jj{97qt5#}fRw*XVg0S-i#IGc})$Syi2hb;t{;t}+4ei0I_I9-K
z&MEirKzpdFof<|%d<Qz+1+3ANKZp3cke@EKOYOXtKaRc0ZSnmwo1gD%`5v`;ejc6s
z`R&^>m45cp$wW_Qmzq|4ywvSc_f!Qw0`pz9Ae{uF_P}C~+M}j1B5zh!nEO~vcP2D<
zIn*{Rhia9xQSp8`hQ?%HR6H0dN1p7huVxLSU~hJbi#|gV&8Ea&@hk}H-gQ=(m(*rV
z<H66mOXX8|@Lu3K+^A>9F=j-yfLJ1F2cFgRC(y<Orr%jde;aMq^UgZ@*HA;u|6@F>
z^`O>hE8gH!xUp7CySoFaQ%LXr6PZU&D6Sh)=i-5MZ~7y>A3K1lL%jC@9L5W^<=2RA
z;CT*Pg_evX^5&e)91*<qKZngg%b5&)0q<bEo~h;3nNidb-9Y;~d~7*Uux6PV7iPP1
zYHNVSh;Atd3GYy_5GE$~$(!47PVRvj_ivcT74^KWp~KXN&Co3NikoN=JvH?E^bFwN
zNcbb!``;2aDt(aaR*ouB`V9&H0I-Js3iKNKri2d$_6K70&w<wf-vO+lcLUs86J*)V
zL6#i{tf5zfpASaqtCG?hV)|neUInb7FNc`_8^GCH2>|)I1;Oq{PPsLyaqnm7xQr)<
z7Xs0zwhx1cv!Ku;q|z`o&|biu)B(5!k-SfP=m^y*trG8`L26ZcXjJx&(iP<p@axJE
zdX#P`$4RF*l@ZF*ca>2(Pd||OEoB0DI4}aZCy=2h=tSV0)OG<<4+kco;qichUNcYt
z{B>~V>G^;Fz8W|L&Q}6Uv<!<+(WmMCz|(*~3p_`kr-tAN=xxDkpmYa+3wn+OU!<4m
ze2{fs3ho5Y6W|%8mEg;$HI8VM-VVM(U!wPdhX8+y-Y=07I)vV6=vBa2=qu2W4E-fI
zdqaN<d|2Xo=uO~uC_~?nQeUT)(A)HNdLi^Jz!yUW@c(h>yWsg6c)mq%hel+K-$$=Y
z`IpeYLzaExto{T%?}F!Lx)u5Wctdysut~yw67~Ty=g*Lhp|C*jM0g9;$tcpOzZoS>
z2{Hh8k*NffKcdhI26D~yuW0e;`j<j-*fH98y_BzRSu$(+@KW}#W8+q?R4^W*;py>d
z!*<ffl0kzd$9e>FOvkgRJ4P=R?W`pV`B};=+m2DBiP<Yg)}e7dZ&EtzSb}tc8SM6^
zhx7WJY1vLbYtyhXTbi5GXA8!lgLyt%atx1#IS=DTS!$|eI{BiJ9<quH`GO%bhFHvJ
z(eDy;CvTZkMnPYam|aol2{IeHX7h!-vs~%*0)rOQwh9J4Ci0FkmNyL=F&Fc~GK+@k
z&?&=#d{H-Z+%?F-Kw!}j4ozFbRtf&YZT2UV75N2(p3e%)wk|t}*7gF+apI6LIGXmD
zjRNw7HfuA1PFtt-qLCZc9X*4z88kX&*j7nojY-QYup8Cw6U)q9;i>NT`bL(r#)7mk
zXE+zf4cpe|404xLHk#EP;+}~GL%=h0&Kp6djTz>gGcUU*g@shxmKYS-c_b+7l;9lP
zapNct()7GA^c>WXi&3Uv7z<KGwd12X8UmwZR1;}vUN`5~lGu%mqdO&=tc8oCrtw5c
z=kZ&|>#<D1wj4UI(5CT)ot`r0N(EhvEG-B`fLz-IlCk8%37H(!A)J{;c}Z7KTTjwp
zE|)Q~mYK6@)N~FWpiwte$#7ES4Kr8p(r=IJPIjIh9D{{vM8TQKlxB;0Y0w|03PmbG
z5{5DUj<KF@+Kz5!jdGG6)<y0~T~zaV0S;LOuYm0IDZ?~GK1*Y$w@ZCATG8q{&Nby_
z{j~Aq9eZtLTUHdOT_Q}8f=CA{eNYH}nKJ&c%O2##mNk!AE(^kMomigKQ9j@vv*xS{
z7ZRGu&Ktb0RQcmmpaN?PczA_@Ro8j1QSqA9t|=bO^0HQMLP8Z-N(BenT*NO#(7cl2
zNLe{Q;pUiy3BmIbvK3>l@RYG&VZJlxq;A_1IjQFh5;<)-@W4k07ZwWnEU#)@6(i-M
z<zeML4rND3;LpN+lr*gJCS_$ykO9mA&7=|)vEMMjqoMCIAoCRQr=vHAc0o!Sz;tlU
zNeKge4q>~r@|$Py&_On&k*m~8zY11cs$7)tH2RCcST!qig)3Axm8E&l%p}ZP;H&i8
zFgXhid1$o(Ef}x#I-sz23tSF%rlaW11It4zTl)lV<P6q_!L^817S9Zz0KOp8U_t*C
zP&p~oHzX~XUfY*;K-G{C9Xz^>kSUx>3bw-T7+My6ylm~V>|GnPCevDpRik!tGuDm|
zoyVY8NblKS9y0o+p{k7COIpLwaR%HWOF?TN#-R&3uF?Tu9i&W*gM;IDv*94)>&IlH
z1vlyiNb;Dh@psb{<aK0ZndssWv~j`A{X+WNF`is2oKXkfbIY~ZPOP1)$U%;j`ZLhY
zb#@sPl9t^*8t1IJ<L$<Ee|dMUaN%om481sd?wxP0z%~4Pt82>l-I1O<?g;kXtuAul
z@~d%8UI$ZjZ!b90?1w+ssx{ddX?k$&#INqJw#3ulLVl}jM`=3=nqNnElT>y+0<Kx<
zg_}tsWn|4>$N*nIU)S2@naMN9&AJ9`31i50z;Sis&&_fXRw@<CqpTbp@+5hNyCZ(5
z^`Y9H(lT6?>tR@RXRry3R}Sg*U3mQYUg5E;Rt}#38rBfjIPdD&%PR}dtQ@l7E}gvg
zT=DXSvzD-iHa)&}jCdq?Cfq4=zN|Yt9jV{9%C*L8#+rC9+-**9-gy>RG`gebF8K@S
z<N3yOB86x5y4vqm8>(yTDC|wbs)gCkqsXf=)$^#iBlkEdwH)JBq9K0rUputr06C9t
z-SXPh?s?+U1={@Qu3awPMM_UaM?u=O!NYng{QS--^B=76vv<692}E;GnU0D~^=jnG
zRMu#Cv+{I>l?R@KciF1Lx{RJY(&_j3i*dU*<=pAw_woYmro)|(I-KUg;8&H7sl>zL
z^Bx4RQpY~94Aox%=BHjcl|iuXtF|)4Ihp#(WZyX0{6gi$%N=l>x3t_zj~^44Hh9_x
z;n&(?TCZ91J+QXWi5mVZy7cXz+&KMo&wsr1o!4LabBcWV6EkOb9Qxj;agGpaiUl=N
z!kVI}^&!AWh!iD;ugRuZ7(dv>8quzgaZe0P@eT>QS%~{%F~%YxjhdKIiPTaEGL4Nj
zS^&Heh(HF6;KbQW3}hvy4nn;;f<ASGZ3besU{FUSsi`qms|Dka#v-xs*7zBv5xV2&
zSx<szauY*EX=((FUw~f}*K46PJ{~u?BmSs_I*cjYY1B}V9Rf8LFV=8GO^xSkd*OJj
zKHe7Zh<C^P@Yf+>e|%gWfvWzd#`vSsdi=b6#Nv7+ZbTySqWptIL>*~rjNspYHZ`IV
zQM}J{AQ9dZ(r!MgnGPf$6Z*nw%PfDo;Zw?b(pDr7m47D<p#2%#Z{Xbb4IH#hWrj2F
z{PvF*|KX*78-MYIKm5_(J(K$^j^=3S=*+N{E%9fpt>FV!!_JvM^z50zTrqF**Qt(A
z&6)9Kt;{!bHgDrYOUpQ=T;9?S^!6T}!6zVp)Y~)O7jyY(&0d+ohqs~IhEGc`<Yw93
zPkO&CGX4_d8e<zT|2Igt=i-os&ox7y9>|jg!${`}a^Fn1+ejPOsNh4&|4{}4I46_C
zw1QtMaOO!XYZbd^i<5Vu&)<R0^6Qj<;#Tu@U|$%(nJ*W!cm41L(Yt=JgEkz%Z#8rl
zp9L2IN3i+K;Jj@DpBWeNJWVIvU+?Ml(0f1k_4>N_iC;40g@wek>Jji5mb9~2$p!TA
zTLGsjSaEry%2z9~)HHf|18t+9w-fGXj%1)4q2GhA3Vd?F<!*a6YFd#zz4!og{|@2J
z(F+A#SOS!?W!?%^Z(%BSH1PIfm8kHZ1+M^ig{K!Qf92+}+YEk=#@~qW7Q_EI<JhNZ
z95>r^Aj=y)19no<r!nAjh_-+YuZ1_>vW#vH+b4X65&JVB^-p+oj-0vu*!>OM-R&Hb
z@wl56-jMMJ-ukF2@o;P$6X#$M9`h##e;pU#nfAZRL6qM6>R%i?zEmt|IJM(rp#J8*
zbZ@g}m|2SttoobJOrPvN+^q58OHMCXrqSQLY}n1mAF7WY!}q>XoGmPCP+;2q%_U(T
zwQ*!o)a~w~f7I8VwTee|yO>_=Yu0d9n7@n@)3epqpi0xq%0_colW~}}#v}Y70<TP*
z>-9H}FM9_G^2ktMSZMBXH8`SVJEP`h>puNC&@AO(+cvT#oNz9CZBPWx)ZnL)n-uv)
z9IVb6_I+wS=xfz*RrFx#&X(BYF=NpvXa$D-&AL5mE?OTqM6*`P4`%r^vA_AUUa$=>
zD5<394&L~#_pI^#SWh_#U^v#}j}RI>+8<u`odHa&&0xM!^{&2e*$)~2-)G=|0EE@O
A_y7O^

literal 0
HcmV?d00001

-- 
1.5.5.6

^ permalink raw reply related	[flat|nested] 8+ messages in thread

* [KVM-AUTOTEST PATCH v2 2/5] [RFC] KVM test: DTM machine deletion tool for WHQL tests
  2010-07-22 10:14 [KVM-AUTOTEST PATCH v2 1/5] [RFC] KVM test: DTM automation program for WHQL tests Michael Goldish
@ 2010-07-22 10:14 ` Michael Goldish
  2010-07-22 10:14   ` [KVM-AUTOTEST PATCH v2 3/5] [RFC] KVM test: add whql_submission test Michael Goldish
  0 siblings, 1 reply; 8+ messages in thread
From: Michael Goldish @ 2010-07-22 10:14 UTC (permalink / raw)
  To: autotest, kvm

This C# program should run on a DTM server.  It's used by the
whql_client_install test to delete the client machine from the server's data
store (if listed there) prior to client installation.  This seems to be
necessary to prevent trouble during testing (like failure to contact the client
machine).

Note: the binary is copied to the server at run time, so it doesn't need to be
packaged in winutils.iso.

Signed-off-by: Michael Goldish <mgoldish@redhat.com>
---
 client/tests/kvm/deps/whql_delete_machine_15.cs  |   82 ++++++++++++++++++++++
 client/tests/kvm/deps/whql_delete_machine_15.exe |  Bin 0 -> 5632 bytes
 2 files changed, 82 insertions(+), 0 deletions(-)
 create mode 100644 client/tests/kvm/deps/whql_delete_machine_15.cs
 create mode 100644 client/tests/kvm/deps/whql_delete_machine_15.exe

diff --git a/client/tests/kvm/deps/whql_delete_machine_15.cs b/client/tests/kvm/deps/whql_delete_machine_15.cs
new file mode 100644
index 0000000..1d78a6d
--- /dev/null
+++ b/client/tests/kvm/deps/whql_delete_machine_15.cs
@@ -0,0 +1,82 @@
+// DTM machine deletion tool
+// Author: Michael Goldish <mgoldish@redhat.com>
+// Based on sample code by Microsoft.
+
+using System;
+using System.Collections.Generic;
+using System.Text.RegularExpressions;
+using Microsoft.DistributedAutomation.DeviceSelection;
+using Microsoft.DistributedAutomation.SqlDataStore;
+
+namespace automate0
+{
+    class AutoJob
+    {
+        static int Main(string[] args)
+        {
+            if (args.Length != 2)
+            {
+                Console.WriteLine("Error: incorrect number of command line arguments");
+                Console.WriteLine("Usage: {0} serverName clientName",
+                    System.Environment.GetCommandLineArgs()[0]);
+                return 1;
+            }
+            string serverName = args[0];
+            string clientName = args[1];
+
+            try
+            {
+                // Initialize DeviceScript and connect to data store
+                Console.WriteLine("Initializing DeviceScript object");
+                DeviceScript script = new DeviceScript();
+                Console.WriteLine("Connecting to data store");
+                script.ConnectToNamedDataStore(serverName);
+
+                // Find the client machine
+                IResourcePool rootPool = script.GetResourcePoolByName("$");
+                Console.WriteLine("Looking for client machine '{0}'", clientName);
+                IResource machine = rootPool.GetResourceByName(clientName);
+                if (machine == null)
+                {
+                    Console.WriteLine("Client machine not found");
+                    return 0;
+                }
+                Console.WriteLine("Client machine '{0}' found ({1}, {2})",
+                    clientName, machine.OperatingSystem, machine.ProcessorArchitecture);
+
+                // Change the client machine's status to 'unsafe'
+                Console.WriteLine("Changing the client machine's status to 'Unsafe'");
+                try
+                {
+                    machine.ChangeResourceStatus("Unsafe");
+                }
+                catch (Exception e)
+                {
+                    Console.WriteLine("Warning: " + e.Message);
+                }
+                while (machine.Status != "Unsafe")
+                {
+                    try
+                    {
+                        machine = rootPool.GetResourceByName(clientName);
+                    }
+                    catch (Exception e)
+                    {
+                        Console.WriteLine("Warning: " + e.Message);
+                    }
+                    System.Threading.Thread.Sleep(1000);
+                }
+
+                // Delete the client machine from datastore
+                Console.WriteLine("Deleting client machine from data store");
+                script.DeleteResource(machine.Id);
+                return 0;
+            }
+            catch (Exception e)
+            {
+                Console.WriteLine("Error: " + e.Message);
+                return 1;
+            }
+        }
+    }
+}
diff --git a/client/tests/kvm/deps/whql_delete_machine_15.exe b/client/tests/kvm/deps/whql_delete_machine_15.exe
new file mode 100644
index 0000000000000000000000000000000000000000..7f571345dd04b1871c5156556791d97ca2564419
GIT binary patch
literal 5632
zcmeHLO>i7X6@If?N$a(>u_XV+mhI8nE=x|dT04dqW7)gX%Fa5HCA5;91X1?vY-=^n
z?98TTR!A`>IwVwaq3oi#_yUKxfEy61NI?My2%KD%V=g$Zf<u6U9PojH;PAbk*&oS{
z%Umh<y1QS$|M$A*b<2~lT&Ff7ilTq|Dbc%_xn<P&=wS`y#QvWr=%<@*_rI%6-0q)U
zvOK*i-Lf<)x@kC$>+40K%bKHGj-Hz;=oPmlQe9o~gQ4o_aiR&WjZXjm>_pw#U5aR%
zwHVPBU@&pJe*}Y$o<Zk9#}o<t2BiGwnby#O=hjA<w}`f?f6X~u1}yXg=q)guA^L!m
zu(4eZ-1z4pQQt<|qt<%Eb;T9K`h^z$lrL6%tfQMTfTm&%{l-c)o#=8(deX#7`IZ6D
zfu2BbX&HE*lEQYu$SI@C=&WlSdP~a?jkYWmc$f2v#=b?Uoj&O&`lORc!y28f{ZDOA
zI!Z4>5dXR(-?Db4_DX=&0i-rA1w-wCcHA6*=J?JdksUYVk)1a;%kS5jJQ$B}k$<kQ
zV;wGzKs+Y*XjFAK0qm0d0d^`4n>yV%P`%Oa=unR|fLE4w^v3@1JNPZX&79p)7g>&D
zLU!T<T^Qttb=K~v%toLYIgQ`R_qNL^th(FXr<fIIC$1FU8E#m*v%9w=VCvY&MEXbg
zmIk^Z1!Fy!l<t9EKtsQhCNwK;F;KBLad$fg_<Cn=Z>M~R?RIWZxf4X{jvb+F_;uVu
zRJ{G~pzlUUV8}1g?Et0^Ft_5LVInx5LEjtjX!{}qJ7D<8XyN>*7Kp2JvwSR-PJJ%@
z{ITa*sGV$#KZ4wWujB0FJT0LgDEQKH${usPiB0`2<K_zV0=5m=q66pV@;S^eVf{9*
zM{T!=nHyNObK4_?O(go4_8h?rb{-p|jw|SD4YaCBOykaY_#1lKK_}=a4rY<25dAVJ
z9dwJ{!uXyV|Ej3}!5F7LZBUETYij%s#tyotv1~lT<6vZ$Qq4Bl63<)V)IP+{t7)C_
zH|UJAeXpNa-`JG{PvGf!>ZTk%vVM%aX%OR9Sp5g>r!UYZ?Vy4ODXZ{VdRZF={DyX%
z^7LKpBz=i~tYt91hmoanfIkG!%XA+Y1E0?XS(J{<0RQ<2W3rKpfTto~fu8xuZsd_I
zrjP{>V=sLT;~wO|py(QHXn({y$YVgzXg~p4vvOkH7o>tIrzYJ}%@(IAH#<2iJU=B?
z1Z8W!dmj5~V6GUxNK?f#U1?iIDy(|GsL)jLiZFeeG%SZwrteBJBu+~}P30`3?6{t9
znI7dtu~se{MO$QjoT*~X7a<Pk3&ur5YNqD+Rz;-7+)CB5g)9iUY?+`32*bBrXGYk@
zih{hRx=@hWKq^|c<*zoWVPMeWc&;tzqO^Q5VL5`von=e9PDMCAofAIfD~3~IQkH|s
z2{vCarBy|wmMLA&UGx#07sgTIJRv{uD-|*e8)?CstvLW^-3vxVlyZh|6p$rB`5EE4
zHED`z*R|Pd>w0vRxtlz#^pI8-{-vp^kg)2M!_8pav~-aw&y`teE+IhEuff{blHrs^
z-9*7R{F+DOE2gL_Cs}Y3Qbt)&@WHd+rs3B&a&}1yqXbJ7Kvb}Ws1nPC%G~Ouydz$#
z8Ju)Q$d`~gTlfvL>HwPKB=K|6Pm+!k?~tihk^~?hIEQLf!S^n}ql31Hk$Eck8K9Q1
zww0s+WPq!zBm`(az75LGVKjgh%Enbn(=~8<O5?H`FMwVF#F|-GAX~>$HV<7Ev{~R|
z`s?TpH0t;U1*(t*IphkX1YA{qyYTr6up9*)9fQXXe!;pf?E|YL0s}mT@`*7VJGC8z
z=Mz}D^lHP}qN1+%QdfB}X-RQ$42LO*@i1wy+l=QyP6zrp$E>#o?@B~3K3bxwyH!d@
zJ-LcHc8#8ewySgu&>(5X7`);=DmNnVf!UbnjeZ@*-thi#PJH;qs~}T{X%5<WHy5D=
z@pZMXsrMo%5*$H(`9#qa?~skoyBF+P>mF~k%~A1rtU&gm`|{NK6HB!H{a60|(!EUP
zC%1oit?%^T?o#Y$uP@9$ef;h>agW6ICfan;+I3Cq?uufJMM={VQC!D~c06<weT+<i
z-g8KegUp}kq$o&zePBr>zzsQofI*CPcWz1~dir|~^$hk5@h=wB!s-qH&n2{de1pwK
zc3hN3^@8g(zHNN3?pHmHS?X_Yz87wwZ^K(~8=gorg<Rpr)AoTk=B}T+b?8sKW`FbL
z2p%&uc5)%-nl&zqo{oa37lhB%##_jiDwf0L&p<6(m|WExd<*lIhZ>+4{94I!^<(Mu
zi3OA-z9hVbumU$`w|Hd%g<Ke(sN+)AQW2)<{qUX2@ZTBM8G5kQ-?L51r7>5IuLyM`
zsoP43RLSPI4gTUeM0!R+PX^9F1n&uz+O=JnPXvEWF(GWh8)^D8@JkQDv*21N-JI0%
zqw7F?FGG~$0~ofCMv49skQF8uIgWWAmGu(Fahd^?$II^m)_II)0dxCh^yB-%Yf&v5
z!<i7m_lECbeak7_ys8*yQ5(2M<WV8G7IHgCEH#U2z!l5`ovV=rs-x(59Y^1WQVl-&
z;0mhCgPPV9Pa3yQ@H>usAq@@BBHtP!2rP4z@>=eXrX3x;X?)gA-g)p!a5s6<c>6Uw
z@7)&gb2Pr<b>O>=d<C~0a*ntr(b=XCSwSo4_~S+SGy%A*B>2{<(yEHCjLMChjM$%y
z(mxgAIdaxtkA+u!n|`*P<0_tM$h)YrHFZaBjH($A$Hp;n4zlo=Z*G1O*ziopxmcf#
zjdBpBkALy&Q)gBxw!SQ+hj-XW@@OiZ)P-ZZ{EQh%&dr`3I+4^pABD}ZT}O;0SA~~6
zbGj>j%J4i<DcY+#6gb{UvL>CAp1C9{hBs8HzyF3zw{p_(Dyik8NgeM+YZ0&J`BrOC
zrRxo4`4Z|V-bU*@!XIGp%E4oABssa7tyXOduP%N^8r5oYIMCqBn&;=8MfWlNIhIs%
zu<Z%6hF9)txCTbzi3UGKX<Ay#cwCo-_n2Crt83NKntD)b%^G_=A(n-$+dPgW4KMF3
zyRQnF)N5AO<QM5ka?!9o5eB7{3_rx1y6eO1d_OhZNCFs64cB)F3*PM~?XfeUhyQ2v
O#mAO-!uWrbf&T)7*;pL_

literal 0
HcmV?d00001

-- 
1.5.5.6

^ permalink raw reply related	[flat|nested] 8+ messages in thread

* [KVM-AUTOTEST PATCH v2 3/5] [RFC] KVM test: add whql_submission test
  2010-07-22 10:14 ` [KVM-AUTOTEST PATCH v2 2/5] [RFC] KVM test: DTM machine deletion tool " Michael Goldish
@ 2010-07-22 10:14   ` Michael Goldish
  2010-07-22 10:14     ` [KVM-AUTOTEST PATCH v2 4/5] [RFC] KVM test: add whql_client_install test Michael Goldish
  2010-08-05  0:49     ` [Autotest] [KVM-AUTOTEST PATCH v2 3/5] [RFC] KVM test: add whql_submission test Amos Kong
  0 siblings, 2 replies; 8+ messages in thread
From: Michael Goldish @ 2010-07-22 10:14 UTC (permalink / raw)
  To: autotest, kvm

whql_submission runs a submission on a given device.  It requires a
functioning external DTM server which runs rss.exe like regular Windows VMs,
preferably with administrator permissions.
The submission is defined by descriptors and device_data objects, which are
specified in the config file(s).  All jobs of the submission are executed.
When all jobs complete, or when the timeout expires, HTML reports are generated
and copied to test.debugdir (client/results/default/kvm...whql_submission/debug)
and the raw test logs (wtl or xml files) are copied to test.debugdir as well.

Changes from v1:
- Send job_filter to the automation program to let it know which tests to
  allow.  job_filter defaults to .*, which means all tests of the submission
  are run.
- Instead of determining test status by the 'pass', 'fail' and 'notrun' values,
  determine it by the 'status' string (e.g. 'Investigate', 'InProgress').
- Kill the client VM if the tests don't complete on time.
- In the final results summary display the job ID of each job.

Signed-off-by: Michael Goldish <mgoldish@redhat.com>
---
 client/tests/kvm/tests/whql_submission.py |  188 +++++++++++++++++++++++++++++
 1 files changed, 188 insertions(+), 0 deletions(-)
 create mode 100644 client/tests/kvm/tests/whql_submission.py

diff --git a/client/tests/kvm/tests/whql_submission.py b/client/tests/kvm/tests/whql_submission.py
new file mode 100644
index 0000000..1fe27c9
--- /dev/null
+++ b/client/tests/kvm/tests/whql_submission.py
@@ -0,0 +1,188 @@
+import logging, time, os, re
+from autotest_lib.client.common_lib import error
+import kvm_subprocess, kvm_test_utils, kvm_utils, rss_file_transfer
+
+
+def run_whql_submission(test, params, env):
+    """
+    WHQL submission test:
+    1) Log into the guest (the client machine) and into a DTM server machine
+    2) Copy the automation program binary (dsso_test_binary) to the server machine
+    3) Run the automation program
+    4) Pass the program all relevant parameters (e.g. device_data)
+    5) Wait for the program to terminate
+    6) Parse and report job results
+    (logs and HTML reports are placed in test.bindir)
+
+    @param test: kvm test object
+    @param params: Dictionary with the test parameters
+    @param env: Dictionary with test environment.
+    """
+    vm = kvm_test_utils.get_living_vm(env, params.get("main_vm"))
+    session = kvm_test_utils.wait_for_login(vm, 0, 240)
+
+    # Collect parameters
+    server_address = params.get("server_address")
+    server_shell_port = int(params.get("server_shell_port"))
+    server_file_transfer_port = int(params.get("server_file_transfer_port"))
+    server_studio_path = params.get("server_studio_path", "%programfiles%\\ "
+                                    "Microsoft Driver Test Manager\\Studio")
+    dsso_test_binary = params.get("dsso_test_binary",
+                                  "deps/whql_submission_15.exe")
+    dsso_test_binary = kvm_utils.get_path(test.bindir, dsso_test_binary)
+    test_device = params.get("test_device")
+    job_filter = params.get("job_filter", ".*")
+    test_timeout = float(params.get("test_timeout", 600))
+    wtt_services = params.get("wtt_services")
+
+    # Restart WTT service(s) on the client
+    logging.info("Restarting WTT services on client")
+    for svc in wtt_services.split():
+        kvm_test_utils.stop_windows_service(session, svc)
+    for svc in wtt_services.split():
+        kvm_test_utils.start_windows_service(session, svc)
+
+    # Copy dsso_test_binary to the server
+    rss_file_transfer.upload(server_address, server_file_transfer_port,
+                             dsso_test_binary, server_studio_path, timeout=60)
+
+    # Open a shell session with the server
+    server_session = kvm_utils.remote_login("nc", server_address,
+                                            server_shell_port, "", "",
+                                            session.prompt, session.linesep)
+
+    # Get the computer names of the server and client
+    cmd = "echo %computername%"
+    server_name = server_session.get_command_output(cmd).strip()
+    client_name = session.get_command_output(cmd).strip()
+    session.close()
+
+    # Run the automation program on the server
+    server_session.get_command_output("cd %s" % server_studio_path)
+    cmd = "%s %s %s %s %s %s" % (os.path.basename(dsso_test_binary),
+                                 server_name,
+                                 client_name,
+                                 "%s_pool" % client_name,
+                                 "%s_submission" % client_name,
+                                 test_timeout)
+    server_session.sendline(cmd)
+
+    # Helper function: wait for a given prompt and raise an exception if an
+    # error occurs
+    def find_prompt(prompt):
+        m, o = server_session.read_until_last_line_matches(
+            [prompt, server_session.prompt], print_func=logging.info,
+            timeout=600)
+        if m != 0:
+            errors = re.findall("^Error:.*$", o, re.I | re.M)
+            if errors:
+                raise error.TestError(errors[0])
+            else:
+                raise error.TestError("Error running automation program: could "
+                                      "not find '%s' prompt" % prompt)
+
+    # Tell the automation program which device to test
+    find_prompt("Device to test:")
+    server_session.sendline(test_device)
+
+    # Tell the automation program which jobs to run
+    find_prompt("Jobs to run:")
+    server_session.sendline(job_filter)
+
+    # Give the automation program all the device data supplied by the user
+    find_prompt("DeviceData name:")
+    for dd in kvm_utils.get_sub_dict_names(params, "device_data"):
+        dd_params = kvm_utils.get_sub_dict(params, dd)
+        if dd_params.get("dd_name") and dd_params.get("dd_data"):
+            server_session.sendline(dd_params.get("dd_name"))
+            server_session.sendline(dd_params.get("dd_data"))
+    server_session.sendline()
+
+    # Give the automation program all the descriptor information supplied by
+    # the user
+    find_prompt("Descriptor path:")
+    for desc in kvm_utils.get_sub_dict_names(params, "descriptors"):
+        desc_params = kvm_utils.get_sub_dict(params, desc)
+        if desc_params.get("desc_path"):
+            server_session.sendline(desc_params.get("desc_path"))
+    server_session.sendline()
+
+    # Wait for the automation program to terminate
+    m, o = server_session.read_up_to_prompt(print_func=logging.info,
+                                            timeout=test_timeout + 300)
+    # (test_timeout + 300 is used here because the automation program is
+    # supposed to terminate cleanly on its own when test_timeout expires)
+    server_session.close()
+
+    # Look for test results in the automation program's output
+    result_summaries = re.findall(r"---- \[.*?\] ----", o, re.DOTALL)
+    if not result_summaries:
+        raise error.TestError("The automation program did not return any "
+                              "results")
+    results = result_summaries[-1].strip("-")
+    results = eval("".join(results.splitlines()))
+
+    # Download logs and HTML reports from the server
+    for i, r in enumerate(results):
+        if "report" in r:
+            try:
+                rss_file_transfer.download(server_address,
+                                           server_file_transfer_port,
+                                           r["report"], test.debugdir)
+            except rss_file_transfer.FileTransferNotFoundError:
+                pass
+        if "logs" in r:
+            try:
+                rss_file_transfer.download(server_address,
+                                           server_file_transfer_port,
+                                           r["logs"], test.debugdir)
+            except rss_file_transfer.FileTransferNotFoundError:
+                pass
+            else:
+                try:
+                    # Create symlinks to test log dirs to make it easier
+                    # to access them (their original names are not human
+                    # readable)
+                    link_name = "logs_%s" % r["report"].split("\\")[-1]
+                    link_name = link_name.replace(" ", "_")
+                    link_name = link_name.replace("/", "_")
+                    os.symlink(r["logs"].split("\\")[-1],
+                               os.path.join(test.debugdir, link_name))
+                except (KeyError, OSError):
+                    pass
+
+    # Print result summary
+    logging.info("")
+    logging.info("Result summary:")
+    name_length = max(len(r.get("job", "")) for r in results)
+    fmt = "%%-6s %%-%ds %%-15s %%-8s %%-8s %%-8s %%-15s" % name_length
+    logging.info(fmt % ("ID", "Job", "Status", "Pass", "Fail", "NotRun",
+                        "NotApplicable"))
+    logging.info(fmt % ("--", "---", "------", "----", "----", "------",
+                        "-------------"))
+    for r in results:
+        logging.info(fmt % (r.get("id"), r.get("job"), r.get("status"),
+                            r.get("pass"), r.get("fail"), r.get("notrun"),
+                            r.get("notapplicable")))
+    logging.info("(see logs and HTML reports in %s)" % test.debugdir)
+
+    # Kill the VM and fail if the automation program did not terminate on time
+    if not m:
+        vm.destroy()
+        raise error.TestFail("The automation program did not terminate "
+                             "on time")
+
+    # Fail if there are failed or incomplete jobs (kill the VM if there are
+    # incomplete jobs)
+    failed_jobs = [r.get("job") for r in results
+                   if r.get("status", "").lower() == "investigate"]
+    running_jobs = [r.get("job") for r in results
+                    if r.get("status", "").lower() == "inprogress"]
+    errors = []
+    if failed_jobs:
+        errors += ["Jobs failed: %s." % failed_jobs]
+    if running_jobs:
+        vm.destroy()
+        errors += ["Jobs did not complete on time: %s." % running_jobs]
+    if errors:
+        raise error.TestFail(" ".join(errors))
-- 
1.5.5.6

^ permalink raw reply related	[flat|nested] 8+ messages in thread

* [KVM-AUTOTEST PATCH v2 4/5] [RFC] KVM test: add whql_client_install test
  2010-07-22 10:14   ` [KVM-AUTOTEST PATCH v2 3/5] [RFC] KVM test: add whql_submission test Michael Goldish
@ 2010-07-22 10:14     ` Michael Goldish
  2010-07-22 10:14       ` [KVM-AUTOTEST PATCH v2 5/5] [RFC] KVM test: add WHQL test definitions to tests_base.cfg.sample Michael Goldish
  2010-08-05  0:49     ` [Autotest] [KVM-AUTOTEST PATCH v2 3/5] [RFC] KVM test: add whql_submission test Amos Kong
  1 sibling, 1 reply; 8+ messages in thread
From: Michael Goldish @ 2010-07-22 10:14 UTC (permalink / raw)
  To: autotest, kvm

whql_client_install installs the DTM client on a guest.  It requires a
functioning external DTM server which runs rss.exe like regular Windows VMs.

Signed-off-by: Michael Goldish <mgoldish@redhat.com>
---
 client/tests/kvm/tests/whql_client_install.py |  110 +++++++++++++++++++++++++
 1 files changed, 110 insertions(+), 0 deletions(-)
 create mode 100644 client/tests/kvm/tests/whql_client_install.py

diff --git a/client/tests/kvm/tests/whql_client_install.py b/client/tests/kvm/tests/whql_client_install.py
new file mode 100644
index 0000000..f46939e
--- /dev/null
+++ b/client/tests/kvm/tests/whql_client_install.py
@@ -0,0 +1,110 @@
+import logging, time, os, re
+from autotest_lib.client.common_lib import error
+import kvm_subprocess, kvm_test_utils, kvm_utils, rss_file_transfer
+
+
+def run_whql_client_install(test, params, env):
+    """
+    WHQL DTM client installation:
+    1) Log into the guest (the client machine) and into a DTM server machine
+    2) Stop the DTM client service (wttsvc) on the client machine
+    3) Delete the client machine from the server's data store
+    4) Rename the client machine (give it a randomly generated name)
+    5) Move the client machine into the server's workgroup
+    6) Reboot the client machine
+    7) Install the DTM client software
+
+    @param test: kvm test object
+    @param params: Dictionary with the test parameters
+    @param env: Dictionary with test environment.
+    """
+    vm = kvm_test_utils.get_living_vm(env, params.get("main_vm"))
+    session = kvm_test_utils.wait_for_login(vm, 0, 240)
+
+    # Collect test params
+    server_address = params.get("server_address")
+    server_shell_port = int(params.get("server_shell_port"))
+    server_file_transfer_port = int(params.get("server_file_transfer_port"))
+    server_studio_path = params.get("server_studio_path", "%programfiles%\\ "
+                                    "Microsoft Driver Test Manager\\Studio")
+    server_username = params.get("server_username")
+    server_password = params.get("server_password")
+    dsso_delete_machine_binary = params.get("dsso_delete_machine_binary",
+                                            "deps/whql_delete_machine_15.exe")
+    dsso_delete_machine_binary = kvm_utils.get_path(test.bindir,
+                                                    dsso_delete_machine_binary)
+    install_timeout = float(params.get("install_timeout", 600))
+    install_cmd = params.get("install_cmd")
+    wtt_services = params.get("wtt_services")
+
+    # Stop WTT service(s) on client
+    for svc in wtt_services.split():
+        kvm_test_utils.stop_windows_service(session, svc)
+
+    # Copy dsso_delete_machine_binary to server
+    rss_file_transfer.upload(server_address, server_file_transfer_port,
+                             dsso_delete_machine_binary, server_studio_path,
+                             timeout=60)
+
+    # Open a shell session with server
+    server_session = kvm_utils.remote_login("nc", server_address,
+                                            server_shell_port, "", "",
+                                            session.prompt, session.linesep)
+
+    # Get server and client information
+    cmd = "echo %computername%"
+    server_name = server_session.get_command_output(cmd).strip()
+    client_name = session.get_command_output(cmd).strip()
+    cmd = "wmic computersystem get workgroup"
+    server_workgroup = server_session.get_command_output(cmd).strip()
+
+    # Delete the client machine from the server's data store (if it's there)
+    server_session.get_command_output("cd %s" % server_studio_path)
+    cmd = "%s %s %s" % (os.path.basename(dsso_delete_machine_binary),
+                        server_name, client_name)
+    server_session.get_command_output(cmd, print_func=logging.info)
+    server_session.close()
+
+    # Rename the client machine
+    client_name = "autotest_%s" % kvm_utils.generate_random_string(4)
+    logging.info("Renaming client machine to '%s'" % client_name)
+    cmd = ('wmic computersystem where name="%%computername%%" rename name="%s"'
+           % client_name)
+    s = session.get_command_status(cmd, timeout=600)
+    if s != 0:
+        raise error.TestError("Could not rename the client machine")
+
+    # Join the server's workgroup
+    logging.info("Joining workgroup '%s'" % server_workgroup)
+    cmd = ('wmic computersystem where name="%%computername%%" call '
+           'joindomainorworkgroup name="%s"' % server_workgroup)
+    s = session.get_command_status(cmd, timeout=600)
+    if s != 0:
+        raise error.TestError("Could not change the client's workgroup")
+
+    # Reboot
+    session = kvm_test_utils.reboot(vm, session)
+
+    # Access shared resources on the server machine
+    logging.info("Attempting to access remote share on server")
+    cmd = r"net use \\%s /user:%s %s" % (server_name, server_username,
+                                         server_password)
+    end_time = time.time() + 120
+    while time.time() < end_time:
+        s = session.get_command_status(cmd)
+        if s == 0:
+            break
+        time.sleep(5)
+    else:
+        raise error.TestError("Could not access server share from client "
+                              "machine")
+
+    # Install
+    logging.info("Installing DTM client (timeout=%ds)", install_timeout)
+    install_cmd = r"cmd /c \\%s\%s" % (server_name, install_cmd.lstrip("\\"))
+    s, o = session.get_command_status_output(install_cmd,
+                                             timeout=install_timeout)
+    if s != 0:
+        raise error.TestError("Client installation failed")
+
+    session.close()
-- 
1.5.5.6

^ permalink raw reply related	[flat|nested] 8+ messages in thread

* [KVM-AUTOTEST PATCH v2 5/5] [RFC] KVM test: add WHQL test definitions to tests_base.cfg.sample
  2010-07-22 10:14     ` [KVM-AUTOTEST PATCH v2 4/5] [RFC] KVM test: add whql_client_install test Michael Goldish
@ 2010-07-22 10:14       ` Michael Goldish
  2010-08-11  3:08         ` [Autotest] " Amos Kong
  0 siblings, 1 reply; 8+ messages in thread
From: Michael Goldish @ 2010-07-22 10:14 UTC (permalink / raw)
  To: autotest, kvm

The parameters that define submissions (dd_name_*, dd_data_*, desc_path_*) were
collected from manually created submissions.
I haven't yet collected the parameters for Windows 2003 and 2008, so for now
WHQL tests are disabled for these versions.

This patch also adds a comment in tests.cfg.sample, encouraging the user to
specify the DTM server's address and ports there.

Note that this patch renames WinVista.32sp1 to WinVista.32.sp1,
                             WinVista.32sp2 to WinVista.32.sp2,
                             WinVista.64sp1 to WinVista.64.sp1 and
                             WinVista.64sp2 to WinVista.64.sp2.

Changes from v1:
- Run all whql_submission tests with -snapshot
- Split whql_submission.hdd into 2 variants: a full one and one that only runs
  a single test (by setting job_filter).  Running all tests in a HDD submission
  takes over 72 hours, so it probably can't be done routinely as part of our
  regression testing.
- Change test_device regex to "qemu harddisk" in whql_submission.hdd.
- Update timeouts
- Enable WHQL tests for Win2003

Signed-off-by: Michael Goldish <mgoldish@redhat.com>
---
 client/tests/kvm/tests.cfg.sample      |    8 +
 client/tests/kvm/tests_base.cfg.sample |  251 +++++++++++++++++++++++++-------
 2 files changed, 205 insertions(+), 54 deletions(-)

diff --git a/client/tests/kvm/tests.cfg.sample b/client/tests/kvm/tests.cfg.sample
index e01406e..cdb7b0a 100644
--- a/client/tests/kvm/tests.cfg.sample
+++ b/client/tests/kvm/tests.cfg.sample
@@ -73,6 +73,14 @@ variants:
         only Fedora.13.64
         only unattended_install.cdrom boot shutdown
 
+# You may provide information about the DTM server for WHQL tests here:
+#whql:
+#    server_address = 10.20.30.40
+#    server_shell_port = 10022
+#    server_file_transfer_port = 10023
+# Note that the DTM server must run rss.exe (available under deps/),
+# preferably with administrator privileges.
+
 # Uncomment the following lines to enable abort-on-error mode:
 #abort_on_error = yes
 #kill_vm.* ?= no
diff --git a/client/tests/kvm/tests_base.cfg.sample b/client/tests/kvm/tests_base.cfg.sample
index d0b8acb..f631184 100644
--- a/client/tests/kvm/tests_base.cfg.sample
+++ b/client/tests/kvm/tests_base.cfg.sample
@@ -285,6 +285,90 @@ variants:
         iozone_cmd = "D:\IOzone\iozone.exe -a"
         iozone_timeout = 3600
 
+    - @whql:         install setup unattended_install.cdrom
+        nic_mode = tap
+        # Replace this with the address of an installed DTM server
+        server_address = 10.20.30.40
+        # The server should run rss.exe like a regular Windows VM, preferably
+        # with administrator privileges (or at least with permission to write
+        # to the DTM studio directory)
+        server_shell_port = 10022
+        server_file_transfer_port = 10023
+        server_studio_path = %programfiles%\Microsoft Driver Test Manager\Studio
+        wtt_services = wttsvc
+        variants:
+            - whql_client_install:
+                type = whql_client_install
+                dsso_delete_machine_binary = deps/whql_delete_machine_15.exe
+                # The username and password are required for accessing the DTM client
+                # installer binary shared by the server
+                server_username = administrator
+                server_password = 1q2w3eP
+                # This path refers to a shared directory on the server
+                # (the final cmd will be something like \\servername\DTMInstall\...)
+                install_cmd = \DTMInstall\Client\Setup.exe /passive
+                install_timeout = 3600
+            - whql_submission:    whql_client_install
+                type = whql_submission
+                extra_params += " -snapshot"
+                dsso_test_binary = deps/whql_submission_15.exe
+                test_timeout = 3600
+                device_data = cat0 cat1 cat2 cat3 logoarch logoos whqlos whqlqual prog desc filter virt
+                descriptors = desc1 desc2 desc3
+                # DeviceData names
+                dd_name_cat0     = Category
+                dd_name_cat1     = Category
+                dd_name_cat2     = Category
+                dd_name_cat3     = Category
+                dd_name_logoarch = LogoProcessorArchitecture
+                dd_name_logoos   = LogoOperatingSystem
+                dd_name_whqlos   = WhqlOs
+                dd_name_whqlqual = WhqlQualification
+                dd_name_prog     = LogoProgramId
+                dd_name_desc     = LogoProgramDescription
+                dd_name_filter   = WDKFilterAttribute
+                dd_name_virt     = ParaVirtualizationDriver
+                # Common DeviceData data
+                dd_data_filter   = FilterIfNoInf
+                dd_data_virt     = True
+                variants:
+                    - keyboard:
+                        # test_device is a regular expression that should match a device's
+                        # name as it appears in device manager.  The first device that matches
+                        # is used.
+                        test_device = keyboard
+                        # Set timeout to 10 hours
+                        test_timeout = 36000
+                        dd_data_cat0 = Input\Keyboard
+                        dd_data_cat1 = Device Fundamentals
+                        dd_data_cat2 = System Fundamentals\Dynamic Partitioning
+                        dd_data_prog = InputKbd
+                        dd_data_desc = Input > Keyboard
+                    - hdd:
+                        test_device = qemu harddisk
+                        device_data += " ex0 ex1 ex2 ex3"
+                        dd_data_cat0 = Storage\Device Class\Disk\Disk
+                        dd_data_cat1 = Storage\Device Class\Disk\Fixed
+                        dd_data_cat2 = Storage\Device Class\Disk\Bus\ATA
+                        dd_data_cat3 = Device Fundamentals
+                        dd_data_prog = StorHDD
+                        dd_data_desc = Storage > Hard Disk Drive (HDD)
+                        dd_name_ex0 = Storage_bus_type
+                        dd_data_ex0 = ATA/ATAPI
+                        dd_name_ex1 = Hybrid_HDD_Support
+                        dd_data_ex1 = 0
+                        dd_name_ex2 = Non_Rotating_Media
+                        dd_data_ex2 = 0
+                        dd_name_ex3 = Secure_Storage
+                        dd_data_ex3 = 0
+                        variants:
+                            - full:
+                                # Yes, 100 hours, this is not a mistake
+                                test_timeout = 360000
+                            - syscache_test:
+                                job_filter = syscache test
+                                test_timeout = 7200
+
     - guest_s4:     install setup unattended_install.cdrom
         type = guest_s4
         check_s4_support_cmd = grep -q disk /sys/power/state
@@ -497,7 +581,7 @@ variants:
 variants:
     # Linux section
     - @Linux:
-        no autoit iozone_windows
+        no autoit iozone_windows whql
         shutdown_command = shutdown -h now
         reboot_command = shutdown -r now
         status_test_command = echo $?
@@ -1100,7 +1184,7 @@ variants:
 
         variants:
             - Win2000:
-                no reboot
+                no reboot whql
                 image_name = win2000-32
                 kill_vm_gracefully = no
                 install:
@@ -1131,6 +1215,13 @@ variants:
                             md5sum_1m = b473bf75af2d1269fec8958cf0202bfd
                             unattended_file = unattended/winxp32.sif
                             floppy = images/winXP-32/floppy.img
+                        whql_submission:
+                            desc_path_desc1 = $\WDK\Logo Type\Device Logo\Windows XP
+                            desc_path_desc2 = $\WDK\Logo Type\Systems Logo\Windows XP
+                            dd_data_logoarch = X86
+                            dd_data_logoos = Windows XP
+                            dd_data_whqlos = Windows XP
+                            dd_data_whqlqual = Basic
 
                     - 64:
                         image_name += -64
@@ -1148,6 +1239,13 @@ variants:
                             md5sum_1m = e812363ff427effc512b7801ee70e513
                             unattended_file = unattended/winxp64.sif
                             floppy = images/winXP-64/floppy.img
+                        whql_submission:
+                            desc_path_desc1 = $\WDK\Logo Type\Device Logo\Windows XP
+                            desc_path_desc2 = $\WDK\Logo Type\Systems Logo\Windows XP
+                            dd_data_logoarch = AMD64
+                            dd_data_logoos = Windows XP 64-Bit Edition Version 2003
+                            dd_data_whqlos = Windows XP x64
+                            dd_data_whqlqual = Basic
 
             - Win2003:
                 image_name = win2003
@@ -1170,6 +1268,12 @@ variants:
                             md5sum_1m = 37c2fdec15ac4ec16aa10fdfdb338aa3
                             unattended_file = unattended/win2003-32.sif
                             floppy = images/win2003-32/floppy.img
+                        whql_submission:
+                            desc_path_desc1 = $\WDK\Logo Type\Device Logo\Windows Server 2003
+                            dd_data_logoarch = X86
+                            dd_data_logoos = Windows Server 2003
+                            dd_data_whqlos = Windows Server 2003
+                            dd_data_whqlqual = Basic
 
                     - 64:
                         image_name += -64
@@ -1187,67 +1291,90 @@ variants:
                             md5sum_1m = 439393c384116aa09e08a0ad047dcea8
                             unattended_file = unattended/win2003-64.sif
                             floppy = images/win2003-64/floppy.img
+                        whql_submission:
+                            desc_path_desc1 = $\WDK\Logo Type\Device Logo\Windows Server 2003
+                            dd_data_logoarch = AMD64
+                            dd_data_logoos = Windows Server 2003
+                            dd_data_whqlos = Windows Server 2003 x64
+                            dd_data_whqlqual = Basic
 
             - WinVista:
                 image_name = winvista
                 image_size = 20G
+                whql_submission:
+                    desc_path_desc1 = $\WDK\Logo Type\Device Logo\Vista Client\Device Premium
+                    desc_path_desc2 = $\WDK\Logo Type\Device Logo\Vista Client\Device Standard
+                    desc_path_desc3 = $\WDK\Logo Type\Device Logo\Vista Client
 
                 variants:
-                    - 32sp1:
-                        image_name += sp1-32
-                        install:
-                            cdrom = windows/WindowsVista-32.iso
-                            md5sum = 1008f323d5170c8e614e52ccb85c0491
-                            md5sum_1m = c724e9695da483bc0fd59e426eaefc72
-                            steps = Win-Vista-32.steps
-                        setup:
-                            steps = WinVista-32-rss.steps
-                        unattended_install.cdrom:
-                            cdrom = windows/WindowsVista-32.iso
-                            md5sum = 1008f323d5170c8e614e52ccb85c0491
-                            md5sum_1m = c724e9695da483bc0fd59e426eaefc72
-                            unattended_file = unattended/winvista-32-autounattend.xml
-                            floppy = images/winvista-sp1-32/floppy.img
-
-                    - 64sp1:
-                        image_name += sp1-64
-                        install:
-                            cdrom = windows/WindowsVista-64.iso
-                            md5sum = 11e2010d857fffc47813295e6be6d58d
-                            md5sum_1m = 0947bcd5390546139e25f25217d6f165
-                            steps = Win-Vista-64.steps
-                        setup:
-                            steps = WinVista-64-rss.steps
-                        unattended_install.cdrom:
-                            cdrom = windows/WindowsVista-64.iso
-                            md5sum = 11e2010d857fffc47813295e6be6d58d
-                            md5sum_1m = 0947bcd5390546139e25f25217d6f165
-                            unattended_file = unattended/winvista-64-autounattend.xml
-                            floppy = images/winvista-sp1-64/floppy.img
-
-                    - 32sp2:
-                        image_name += sp2-32
-                        unattended_install.cdrom:
-                            cdrom = windows/en_windows_vista_with_sp2_x86_dvd_342266.iso
-                            md5sum = 19ca90a425667812977bab6f4ce24175
-                            md5sum_1m = 89c15020e0e6125be19acf7a2e5dc614
-                            sha1sum = 25ad9a776503e6a583bec07879dbcc5dfd20cd6e
-                            sha1sum_1m = a2afa4cffdc1c362dbf9e62942337f4f875a22cf
-                            unattended_file = unattended/winvista-32-autounattend.xml
-                            floppy = images/winvista-sp2-32/floppy.img
+                    - 32:
+                        whql_submission:
+                            dd_data_logoarch = X86
+                            dd_data_logoos = Windows Vista
+                            dd_data_whqlos = Windows Vista Client
+                            dd_data_whqlqual = Premium
+                        variants:
+                            - sp1:
+                                image_name += sp1-32
+                                install:
+                                    cdrom = windows/WindowsVista-32.iso
+                                    md5sum = 1008f323d5170c8e614e52ccb85c0491
+                                    md5sum_1m = c724e9695da483bc0fd59e426eaefc72
+                                    steps = Win-Vista-32.steps
+                                setup:
+                                    steps = WinVista-32-rss.steps
+                                unattended_install.cdrom:
+                                    cdrom = windows/WindowsVista-32.iso
+                                    md5sum = 1008f323d5170c8e614e52ccb85c0491
+                                    md5sum_1m = c724e9695da483bc0fd59e426eaefc72
+                                    unattended_file = unattended/winvista-32-autounattend.xml
+                                    floppy = images/winvista-sp1-32/floppy.img
+                            - sp2:
+                                image_name += sp2-32
+                                unattended_install.cdrom:
+                                    cdrom = windows/en_windows_vista_with_sp2_x86_dvd_342266.iso
+                                    md5sum = 19ca90a425667812977bab6f4ce24175
+                                    md5sum_1m = 89c15020e0e6125be19acf7a2e5dc614
+                                    sha1sum = 25ad9a776503e6a583bec07879dbcc5dfd20cd6e
+                                    sha1sum_1m = a2afa4cffdc1c362dbf9e62942337f4f875a22cf
+                                    unattended_file = unattended/winvista-32-autounattend.xml
+                                    floppy = images/winvista-sp2-32/floppy.img
 
-                    - 64sp2:
-                        image_name += sp2-64
-                        unattended_install.cdrom:
-                            cdrom = windows/en_windows_vista_sp2_x64_dvd_342267.iso
-                            md5sum = a1c024d7abaf34bac3368e88efbc2574
-                            md5sum_1m = 3d84911a80f3df71d1026f7adedc2181
-                            sha1sum = aaee3c04533899f9f8c4ae0c4250ef5fafbe29a3
-                            sha1sum_1m = 1fd21bd3ce2a4de8856c7b8fe6fdf80260f6d1c7
-                            unattended_file = unattended/winvista-64-autounattend.xml
-                            floppy = images/winvista-sp2-64/floppy.img
+                    - 64:
+                        whql_submission:
+                            dd_data_logoarch = AMD64
+                            dd_data_logoos = Windows Vista
+                            dd_data_whqlos = Windows Vista Client x64
+                            dd_data_whqlqual = Premium
+                        variants:
+                            - sp1:
+                                image_name += sp1-64
+                                install:
+                                    cdrom = windows/WindowsVista-64.iso
+                                    md5sum = 11e2010d857fffc47813295e6be6d58d
+                                    md5sum_1m = 0947bcd5390546139e25f25217d6f165
+                                    steps = Win-Vista-64.steps
+                                setup:
+                                    steps = WinVista-64-rss.steps
+                                unattended_install.cdrom:
+                                    cdrom = windows/WindowsVista-64.iso
+                                    md5sum = 11e2010d857fffc47813295e6be6d58d
+                                    md5sum_1m = 0947bcd5390546139e25f25217d6f165
+                                    unattended_file = unattended/winvista-64-autounattend.xml
+                                    floppy = images/winvista-sp1-64/floppy.img
+                            - sp2:
+                                image_name += sp2-64
+                                unattended_install.cdrom:
+                                    cdrom = windows/en_windows_vista_sp2_x64_dvd_342267.iso
+                                    md5sum = a1c024d7abaf34bac3368e88efbc2574
+                                    md5sum_1m = 3d84911a80f3df71d1026f7adedc2181
+                                    sha1sum = aaee3c04533899f9f8c4ae0c4250ef5fafbe29a3
+                                    sha1sum_1m = 1fd21bd3ce2a4de8856c7b8fe6fdf80260f6d1c7
+                                    unattended_file = unattended/winvista-64-autounattend.xml
+                                    floppy = images/winvista-sp2-64/floppy.img
 
             - Win2008:
+                no whql
                 image_name = win2008
                 image_size = 20G
 
@@ -1325,6 +1452,12 @@ variants:
             - Win7:
                 image_name = win7
                 image_size = 20G
+                whql_submission:
+                    desc_path_desc1 = $\WDK\Logo Type\Device Logo\Windows 7 Client\Logo
+                    desc_path_desc2 = $\WDK\Logo Type\Device Logo\Windows 7 Client
+                    device_data += " adq"
+                    dd_name_adq = AdditionalQualificationGroup
+                    dd_data_adq = Windows 7
 
                 variants:
                     - 32:
@@ -1337,6 +1470,11 @@ variants:
                             sha1sum_1m = 9f9c3780aebeb28a9bf22188eed6bc15475dc9c5
                             unattended_file = unattended/win7-32-autounattend.xml
                             floppy = images/win7-32/floppy.img
+                        whql_submission:
+                            dd_data_logoarch = X86
+                            dd_data_logoos = Windows 7
+                            dd_data_whqlos = Windows 7 Client
+                            dd_data_whqlqual = Logo
 
                     - 64:
                         image_name += -64
@@ -1356,6 +1494,11 @@ variants:
                             sha1sum_1m = 4a3903bd5157de54f0702e5263e0a683c5775515
                             unattended_file = unattended/win7-64-autounattend.xml
                             floppy = images/win7-64/floppy.img
+                        whql_submission:
+                            dd_data_logoarch = AMD64
+                            dd_data_logoos = Windows 7
+                            dd_data_whqlos = Windows 7 Client x64
+                            dd_data_whqlqual = Logo
 
 
     # Unix/BSD section
-- 
1.5.5.6

^ permalink raw reply related	[flat|nested] 8+ messages in thread

* Re: [Autotest] [KVM-AUTOTEST PATCH v2 3/5] [RFC] KVM test: add whql_submission test
  2010-07-22 10:14   ` [KVM-AUTOTEST PATCH v2 3/5] [RFC] KVM test: add whql_submission test Michael Goldish
  2010-07-22 10:14     ` [KVM-AUTOTEST PATCH v2 4/5] [RFC] KVM test: add whql_client_install test Michael Goldish
@ 2010-08-05  0:49     ` Amos Kong
  2010-08-05 11:34       ` Michael Goldish
  1 sibling, 1 reply; 8+ messages in thread
From: Amos Kong @ 2010-08-05  0:49 UTC (permalink / raw)
  To: Michael Goldish; +Cc: autotest, kvm

On Thu, Jul 22, 2010 at 01:14:17PM +0300, Michael Goldish wrote:
> whql_submission runs a submission on a given device.  It requires a
> functioning external DTM server which runs rss.exe like regular Windows VMs,
> preferably with administrator permissions.
> The submission is defined by descriptors and device_data objects, which are
> specified in the config file(s).  All jobs of the submission are executed.
> When all jobs complete, or when the timeout expires, HTML reports are generated
> and copied to test.debugdir (client/results/default/kvm...whql_submission/debug)
> and the raw test logs (wtl or xml files) are copied to test.debugdir as well.
> 
> Changes from v1:
> - Send job_filter to the automation program to let it know which tests to
>   allow.  job_filter defaults to .*, which means all tests of the submission
>   are run.
> - Instead of determining test status by the 'pass', 'fail' and 'notrun' values,
>   determine it by the 'status' string (e.g. 'Investigate', 'InProgress').
> - Kill the client VM if the tests don't complete on time.
> - In the final results summary display the job ID of each job.
> 
> Signed-off-by: Michael Goldish <mgoldish@redhat.com>
> ---
>  client/tests/kvm/tests/whql_submission.py |  188 +++++++++++++++++++++++++++++
>  1 files changed, 188 insertions(+), 0 deletions(-)
>  create mode 100644 client/tests/kvm/tests/whql_submission.py
> 
> diff --git a/client/tests/kvm/tests/whql_submission.py b/client/tests/kvm/tests/whql_submission.py
> new file mode 100644
> index 0000000..1fe27c9
> --- /dev/null
> +++ b/client/tests/kvm/tests/whql_submission.py
> @@ -0,0 +1,188 @@
> +import logging, time, os, re
> +from autotest_lib.client.common_lib import error
> +import kvm_subprocess, kvm_test_utils, kvm_utils, rss_file_transfer
> +
> +
> +def run_whql_submission(test, params, env):
> +    """
> +    WHQL submission test:
> +    1) Log into the guest (the client machine) and into a DTM server machine
> +    2) Copy the automation program binary (dsso_test_binary) to the server machine
> +    3) Run the automation program
> +    4) Pass the program all relevant parameters (e.g. device_data)
> +    5) Wait for the program to terminate
> +    6) Parse and report job results
> +    (logs and HTML reports are placed in test.bindir)
> +
> +    @param test: kvm test object
> +    @param params: Dictionary with the test parameters
> +    @param env: Dictionary with test environment.
> +    """
> +    vm = kvm_test_utils.get_living_vm(env, params.get("main_vm"))
> +    session = kvm_test_utils.wait_for_login(vm, 0, 240)

Make the login timeout can be configured.

       timeout = float(params.get("login_timeout", 240))
       session = kvm_test_utils.wait_for_login(vm, 0, timeout, ..)

> +
> +    # Collect parameters
> +    server_address = params.get("server_address")
> +    server_shell_port = int(params.get("server_shell_port"))
> +    server_file_transfer_port = int(params.get("server_file_transfer_port"))
> +    server_studio_path = params.get("server_studio_path", "%programfiles%\\ "
> +                                    "Microsoft Driver Test Manager\\Studio")
> +    dsso_test_binary = params.get("dsso_test_binary",
> +                                  "deps/whql_submission_15.exe")
> +    dsso_test_binary = kvm_utils.get_path(test.bindir, dsso_test_binary)
> +    test_device = params.get("test_device")
> +    job_filter = params.get("job_filter", ".*")
> +    test_timeout = float(params.get("test_timeout", 600))
> +    wtt_services = params.get("wtt_services")
> +
> +    # Restart WTT service(s) on the client
> +    logging.info("Restarting WTT services on client")
> +    for svc in wtt_services.split():
> +        kvm_test_utils.stop_windows_service(session, svc)
> +    for svc in wtt_services.split():
> +        kvm_test_utils.start_windows_service(session, svc)
> +
> +    # Copy dsso_test_binary to the server
> +    rss_file_transfer.upload(server_address, server_file_transfer_port,
> +                             dsso_test_binary, server_studio_path, timeout=60)
> +
> +    # Open a shell session with the server
> +    server_session = kvm_utils.remote_login("nc", server_address,
> +                                            server_shell_port, "", "",
> +                                            session.prompt, session.linesep)

login_timeout issue

> +    # Get the computer names of the server and client
> +    cmd = "echo %computername%"
> +    server_name = server_session.get_command_output(cmd).strip()
> +    client_name = session.get_command_output(cmd).strip()
> +    session.close()
> +
> +    # Run the automation program on the server
> +    server_session.get_command_output("cd %s" % server_studio_path)
> +    cmd = "%s %s %s %s %s %s" % (os.path.basename(dsso_test_binary),
> +                                 server_name,
> +                                 client_name,
> +                                 "%s_pool" % client_name,
> +                                 "%s_submission" % client_name,
> +                                 test_timeout)
> +    server_session.sendline(cmd)
> +
> +    # Helper function: wait for a given prompt and raise an exception if an
> +    # error occurs
> +    def find_prompt(prompt):
> +        m, o = server_session.read_until_last_line_matches(
> +            [prompt, server_session.prompt], print_func=logging.info,
> +            timeout=600)
> +        if m != 0:
> +            errors = re.findall("^Error:.*$", o, re.I | re.M)
> +            if errors:
> +                raise error.TestError(errors[0])
> +            else:
> +                raise error.TestError("Error running automation program: could "
> +                                      "not find '%s' prompt" % prompt)
> +
> +    # Tell the automation program which device to test
> +    find_prompt("Device to test:")
> +    server_session.sendline(test_device)
> +
> +    # Tell the automation program which jobs to run
> +    find_prompt("Jobs to run:")
> +    server_session.sendline(job_filter)
> +
> +    # Give the automation program all the device data supplied by the user
> +    find_prompt("DeviceData name:")
> +    for dd in kvm_utils.get_sub_dict_names(params, "device_data"):
> +        dd_params = kvm_utils.get_sub_dict(params, dd)
> +        if dd_params.get("dd_name") and dd_params.get("dd_data"):
> +            server_session.sendline(dd_params.get("dd_name"))
> +            server_session.sendline(dd_params.get("dd_data"))
> +    server_session.sendline()

What's the purpose of this sentence ?
> +
> +    # Give the automation program all the descriptor information supplied by
> +    # the user
> +    find_prompt("Descriptor path:")
> +    for desc in kvm_utils.get_sub_dict_names(params, "descriptors"):
> +        desc_params = kvm_utils.get_sub_dict(params, desc)
> +        if desc_params.get("desc_path"):
> +            server_session.sendline(desc_params.get("desc_path"))
> +    server_session.sendline()
> +
> +    # Wait for the automation program to terminate
> +    m, o = server_session.read_up_to_prompt(print_func=logging.info,
> +                                            timeout=test_timeout + 300)
> +    # (test_timeout + 300 is used here because the automation program is
> +    # supposed to terminate cleanly on its own when test_timeout expires)
> +    server_session.close()
> +
> +    # Look for test results in the automation program's output
> +    result_summaries = re.findall(r"---- \[.*?\] ----", o, re.DOTALL)
> +    if not result_summaries:
> +        raise error.TestError("The automation program did not return any "
> +                              "results")
> +    results = result_summaries[-1].strip("-")
> +    results = eval("".join(results.splitlines()))
> +
> +    # Download logs and HTML reports from the server
> +    for i, r in enumerate(results):
> +        if "report" in r:
> +            try:
> +                rss_file_transfer.download(server_address,
> +                                           server_file_transfer_port,
> +                                           r["report"], test.debugdir)
> +            except rss_file_transfer.FileTransferNotFoundError:
> +                pass
> +        if "logs" in r:
> +            try:
> +                rss_file_transfer.download(server_address,
> +                                           server_file_transfer_port,
> +                                           r["logs"], test.debugdir)
> +            except rss_file_transfer.FileTransferNotFoundError:
> +                pass
> +            else:
> +                try:
> +                    # Create symlinks to test log dirs to make it easier
> +                    # to access them (their original names are not human
> +                    # readable)
> +                    link_name = "logs_%s" % r["report"].split("\\")[-1]
> +                    link_name = link_name.replace(" ", "_")
> +                    link_name = link_name.replace("/", "_")
> +                    os.symlink(r["logs"].split("\\")[-1],
> +                               os.path.join(test.debugdir, link_name))
> +                except (KeyError, OSError):
> +                    pass
> +
> +    # Print result summary
> +    logging.info("")
> +    logging.info("Result summary:")
> +    name_length = max(len(r.get("job", "")) for r in results)
> +    fmt = "%%-6s %%-%ds %%-15s %%-8s %%-8s %%-8s %%-15s" % name_length
> +    logging.info(fmt % ("ID", "Job", "Status", "Pass", "Fail", "NotRun",
> +                        "NotApplicable"))
> +    logging.info(fmt % ("--", "---", "------", "----", "----", "------",
> +                        "-------------"))
> +    for r in results:
> +        logging.info(fmt % (r.get("id"), r.get("job"), r.get("status"),
> +                            r.get("pass"), r.get("fail"), r.get("notrun"),
> +                            r.get("notapplicable")))
> +    logging.info("(see logs and HTML reports in %s)" % test.debugdir)
> +
> +    # Kill the VM and fail if the automation program did not terminate on time
> +    if not m:
> +        vm.destroy()
> +        raise error.TestFail("The automation program did not terminate "
> +                             "on time")
> +
> +    # Fail if there are failed or incomplete jobs (kill the VM if there are
> +    # incomplete jobs)
> +    failed_jobs = [r.get("job") for r in results
> +                   if r.get("status", "").lower() == "investigate"]
> +    running_jobs = [r.get("job") for r in results
> +                    if r.get("status", "").lower() == "inprogress"]
> +    errors = []
> +    if failed_jobs:
> +        errors += ["Jobs failed: %s." % failed_jobs]
> +    if running_jobs:
> +        vm.destroy()
> +        errors += ["Jobs did not complete on time: %s." % running_jobs]
> +    if errors:
> +        raise error.TestFail(" ".join(errors))
> -- 
> 1.5.5.6
> 
> _______________________________________________
> Autotest mailing list
> Autotest@test.kernel.org
> http://test.kernel.org/cgi-bin/mailman/listinfo/autotest

^ permalink raw reply	[flat|nested] 8+ messages in thread

* Re: [Autotest] [KVM-AUTOTEST PATCH v2 3/5] [RFC] KVM test: add whql_submission test
  2010-08-05  0:49     ` [Autotest] [KVM-AUTOTEST PATCH v2 3/5] [RFC] KVM test: add whql_submission test Amos Kong
@ 2010-08-05 11:34       ` Michael Goldish
  0 siblings, 0 replies; 8+ messages in thread
From: Michael Goldish @ 2010-08-05 11:34 UTC (permalink / raw)
  To: Amos Kong; +Cc: autotest, kvm

On 08/05/2010 03:49 AM, Amos Kong wrote:
> On Thu, Jul 22, 2010 at 01:14:17PM +0300, Michael Goldish wrote:
>> whql_submission runs a submission on a given device.  It requires a
>> functioning external DTM server which runs rss.exe like regular Windows VMs,
>> preferably with administrator permissions.
>> The submission is defined by descriptors and device_data objects, which are
>> specified in the config file(s).  All jobs of the submission are executed.
>> When all jobs complete, or when the timeout expires, HTML reports are generated
>> and copied to test.debugdir (client/results/default/kvm...whql_submission/debug)
>> and the raw test logs (wtl or xml files) are copied to test.debugdir as well.
>>
>> Changes from v1:
>> - Send job_filter to the automation program to let it know which tests to
>>   allow.  job_filter defaults to .*, which means all tests of the submission
>>   are run.
>> - Instead of determining test status by the 'pass', 'fail' and 'notrun' values,
>>   determine it by the 'status' string (e.g. 'Investigate', 'InProgress').
>> - Kill the client VM if the tests don't complete on time.
>> - In the final results summary display the job ID of each job.
>>
>> Signed-off-by: Michael Goldish <mgoldish@redhat.com>
>> ---
>>  client/tests/kvm/tests/whql_submission.py |  188 +++++++++++++++++++++++++++++
>>  1 files changed, 188 insertions(+), 0 deletions(-)
>>  create mode 100644 client/tests/kvm/tests/whql_submission.py
>>
>> diff --git a/client/tests/kvm/tests/whql_submission.py b/client/tests/kvm/tests/whql_submission.py
>> new file mode 100644
>> index 0000000..1fe27c9
>> --- /dev/null
>> +++ b/client/tests/kvm/tests/whql_submission.py
>> @@ -0,0 +1,188 @@
>> +import logging, time, os, re
>> +from autotest_lib.client.common_lib import error
>> +import kvm_subprocess, kvm_test_utils, kvm_utils, rss_file_transfer
>> +
>> +
>> +def run_whql_submission(test, params, env):
>> +    """
>> +    WHQL submission test:
>> +    1) Log into the guest (the client machine) and into a DTM server machine
>> +    2) Copy the automation program binary (dsso_test_binary) to the server machine
>> +    3) Run the automation program
>> +    4) Pass the program all relevant parameters (e.g. device_data)
>> +    5) Wait for the program to terminate
>> +    6) Parse and report job results
>> +    (logs and HTML reports are placed in test.bindir)
>> +
>> +    @param test: kvm test object
>> +    @param params: Dictionary with the test parameters
>> +    @param env: Dictionary with test environment.
>> +    """
>> +    vm = kvm_test_utils.get_living_vm(env, params.get("main_vm"))
>> +    session = kvm_test_utils.wait_for_login(vm, 0, 240)
> 
> Make the login timeout can be configured.
> 
>        timeout = float(params.get("login_timeout", 240))
>        session = kvm_test_utils.wait_for_login(vm, 0, timeout, ..)

I agree, will fix that.

>> +
>> +    # Collect parameters
>> +    server_address = params.get("server_address")
>> +    server_shell_port = int(params.get("server_shell_port"))
>> +    server_file_transfer_port = int(params.get("server_file_transfer_port"))
>> +    server_studio_path = params.get("server_studio_path", "%programfiles%\\ "
>> +                                    "Microsoft Driver Test Manager\\Studio")
>> +    dsso_test_binary = params.get("dsso_test_binary",
>> +                                  "deps/whql_submission_15.exe")
>> +    dsso_test_binary = kvm_utils.get_path(test.bindir, dsso_test_binary)
>> +    test_device = params.get("test_device")
>> +    job_filter = params.get("job_filter", ".*")
>> +    test_timeout = float(params.get("test_timeout", 600))
>> +    wtt_services = params.get("wtt_services")
>> +
>> +    # Restart WTT service(s) on the client
>> +    logging.info("Restarting WTT services on client")
>> +    for svc in wtt_services.split():
>> +        kvm_test_utils.stop_windows_service(session, svc)
>> +    for svc in wtt_services.split():
>> +        kvm_test_utils.start_windows_service(session, svc)
>> +
>> +    # Copy dsso_test_binary to the server
>> +    rss_file_transfer.upload(server_address, server_file_transfer_port,
>> +                             dsso_test_binary, server_studio_path, timeout=60)
>> +
>> +    # Open a shell session with the server
>> +    server_session = kvm_utils.remote_login("nc", server_address,
>> +                                            server_shell_port, "", "",
>> +                                            session.prompt, session.linesep)
> 
> login_timeout issue

It's a session with an external server, not an autotest VM, and I think
it's fair to expect the server to be up and running during testing.  10
seconds (the default) should be more than enough for the server to
reply, but still, if you think it's necessary, I can make this timeout
configurable.

>> +    # Get the computer names of the server and client
>> +    cmd = "echo %computername%"
>> +    server_name = server_session.get_command_output(cmd).strip()
>> +    client_name = session.get_command_output(cmd).strip()
>> +    session.close()
>> +
>> +    # Run the automation program on the server
>> +    server_session.get_command_output("cd %s" % server_studio_path)
>> +    cmd = "%s %s %s %s %s %s" % (os.path.basename(dsso_test_binary),
>> +                                 server_name,
>> +                                 client_name,
>> +                                 "%s_pool" % client_name,
>> +                                 "%s_submission" % client_name,
>> +                                 test_timeout)
>> +    server_session.sendline(cmd)
>> +
>> +    # Helper function: wait for a given prompt and raise an exception if an
>> +    # error occurs
>> +    def find_prompt(prompt):
>> +        m, o = server_session.read_until_last_line_matches(
>> +            [prompt, server_session.prompt], print_func=logging.info,
>> +            timeout=600)
>> +        if m != 0:
>> +            errors = re.findall("^Error:.*$", o, re.I | re.M)
>> +            if errors:
>> +                raise error.TestError(errors[0])
>> +            else:
>> +                raise error.TestError("Error running automation program: could "
>> +                                      "not find '%s' prompt" % prompt)
>> +
>> +    # Tell the automation program which device to test
>> +    find_prompt("Device to test:")
>> +    server_session.sendline(test_device)
>> +
>> +    # Tell the automation program which jobs to run
>> +    find_prompt("Jobs to run:")
>> +    server_session.sendline(job_filter)
>> +
>> +    # Give the automation program all the device data supplied by the user
>> +    find_prompt("DeviceData name:")
>> +    for dd in kvm_utils.get_sub_dict_names(params, "device_data"):
>> +        dd_params = kvm_utils.get_sub_dict(params, dd)
>> +        if dd_params.get("dd_name") and dd_params.get("dd_data"):
>> +            server_session.sendline(dd_params.get("dd_name"))
>> +            server_session.sendline(dd_params.get("dd_data"))
>> +    server_session.sendline()
> 
> What's the purpose of this sentence ?

If you mean the empty sendline(), it tells the automation program that
we're done specifying DeviceData objects.

>> +
>> +    # Give the automation program all the descriptor information supplied by
>> +    # the user
>> +    find_prompt("Descriptor path:")
>> +    for desc in kvm_utils.get_sub_dict_names(params, "descriptors"):
>> +        desc_params = kvm_utils.get_sub_dict(params, desc)
>> +        if desc_params.get("desc_path"):
>> +            server_session.sendline(desc_params.get("desc_path"))
>> +    server_session.sendline()

Same here: this tells the program we're done with descriptors.

>> +    # Wait for the automation program to terminate
>> +    m, o = server_session.read_up_to_prompt(print_func=logging.info,
>> +                                            timeout=test_timeout + 300)
>> +    # (test_timeout + 300 is used here because the automation program is
>> +    # supposed to terminate cleanly on its own when test_timeout expires)
>> +    server_session.close()
>> +
>> +    # Look for test results in the automation program's output
>> +    result_summaries = re.findall(r"---- \[.*?\] ----", o, re.DOTALL)
>> +    if not result_summaries:
>> +        raise error.TestError("The automation program did not return any "
>> +                              "results")
>> +    results = result_summaries[-1].strip("-")
>> +    results = eval("".join(results.splitlines()))
>> +
>> +    # Download logs and HTML reports from the server
>> +    for i, r in enumerate(results):
>> +        if "report" in r:
>> +            try:
>> +                rss_file_transfer.download(server_address,
>> +                                           server_file_transfer_port,
>> +                                           r["report"], test.debugdir)
>> +            except rss_file_transfer.FileTransferNotFoundError:
>> +                pass
>> +        if "logs" in r:
>> +            try:
>> +                rss_file_transfer.download(server_address,
>> +                                           server_file_transfer_port,
>> +                                           r["logs"], test.debugdir)
>> +            except rss_file_transfer.FileTransferNotFoundError:
>> +                pass
>> +            else:
>> +                try:
>> +                    # Create symlinks to test log dirs to make it easier
>> +                    # to access them (their original names are not human
>> +                    # readable)
>> +                    link_name = "logs_%s" % r["report"].split("\\")[-1]
>> +                    link_name = link_name.replace(" ", "_")
>> +                    link_name = link_name.replace("/", "_")
>> +                    os.symlink(r["logs"].split("\\")[-1],
>> +                               os.path.join(test.debugdir, link_name))
>> +                except (KeyError, OSError):
>> +                    pass
>> +
>> +    # Print result summary
>> +    logging.info("")
>> +    logging.info("Result summary:")
>> +    name_length = max(len(r.get("job", "")) for r in results)
>> +    fmt = "%%-6s %%-%ds %%-15s %%-8s %%-8s %%-8s %%-15s" % name_length
>> +    logging.info(fmt % ("ID", "Job", "Status", "Pass", "Fail", "NotRun",
>> +                        "NotApplicable"))
>> +    logging.info(fmt % ("--", "---", "------", "----", "----", "------",
>> +                        "-------------"))
>> +    for r in results:
>> +        logging.info(fmt % (r.get("id"), r.get("job"), r.get("status"),
>> +                            r.get("pass"), r.get("fail"), r.get("notrun"),
>> +                            r.get("notapplicable")))
>> +    logging.info("(see logs and HTML reports in %s)" % test.debugdir)
>> +
>> +    # Kill the VM and fail if the automation program did not terminate on time
>> +    if not m:
>> +        vm.destroy()
>> +        raise error.TestFail("The automation program did not terminate "
>> +                             "on time")
>> +
>> +    # Fail if there are failed or incomplete jobs (kill the VM if there are
>> +    # incomplete jobs)
>> +    failed_jobs = [r.get("job") for r in results
>> +                   if r.get("status", "").lower() == "investigate"]
>> +    running_jobs = [r.get("job") for r in results
>> +                    if r.get("status", "").lower() == "inprogress"]
>> +    errors = []
>> +    if failed_jobs:
>> +        errors += ["Jobs failed: %s." % failed_jobs]
>> +    if running_jobs:
>> +        vm.destroy()
>> +        errors += ["Jobs did not complete on time: %s." % running_jobs]
>> +    if errors:
>> +        raise error.TestFail(" ".join(errors))
>> -- 
>> 1.5.5.6
>>
>> _______________________________________________
>> Autotest mailing list
>> Autotest@test.kernel.org
>> http://test.kernel.org/cgi-bin/mailman/listinfo/autotest
> --
> To unsubscribe from this list: send the line "unsubscribe kvm" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html


^ permalink raw reply	[flat|nested] 8+ messages in thread

* Re: [Autotest] [KVM-AUTOTEST PATCH v2 5/5] [RFC] KVM test: add WHQL test definitions to tests_base.cfg.sample
  2010-07-22 10:14       ` [KVM-AUTOTEST PATCH v2 5/5] [RFC] KVM test: add WHQL test definitions to tests_base.cfg.sample Michael Goldish
@ 2010-08-11  3:08         ` Amos Kong
  0 siblings, 0 replies; 8+ messages in thread
From: Amos Kong @ 2010-08-11  3:08 UTC (permalink / raw)
  To: Michael Goldish; +Cc: autotest, kvm

On Thu, Jul 22, 2010 at 01:14:19PM +0300, Michael Goldish wrote:
> The parameters that define submissions (dd_name_*, dd_data_*, desc_path_*) were
> collected from manually created submissions.
> I haven't yet collected the parameters for Windows 2003 and 2008, so for now
> WHQL tests are disabled for these versions.
> 
> This patch also adds a comment in tests.cfg.sample, encouraging the user to
> specify the DTM server's address and ports there.
> 
> Note that this patch renames WinVista.32sp1 to WinVista.32.sp1,
>                              WinVista.32sp2 to WinVista.32.sp2,
>                              WinVista.64sp1 to WinVista.64.sp1 and
>                              WinVista.64sp2 to WinVista.64.sp2.
> 
> Changes from v1:
> - Run all whql_submission tests with -snapshot
> - Split whql_submission.hdd into 2 variants: a full one and one that only runs
>   a single test (by setting job_filter).  Running all tests in a HDD submission
>   takes over 72 hours, so it probably can't be done routinely as part of our
>   regression testing.
> - Change test_device regex to "qemu harddisk" in whql_submission.hdd.
> - Update timeouts
> - Enable WHQL tests for Win2003
> 
> Signed-off-by: Michael Goldish <mgoldish@redhat.com>
> ---
>  client/tests/kvm/tests.cfg.sample      |    8 +
>  client/tests/kvm/tests_base.cfg.sample |  251 +++++++++++++++++++++++++-------
>  2 files changed, 205 insertions(+), 54 deletions(-)
> 
> diff --git a/client/tests/kvm/tests.cfg.sample b/client/tests/kvm/tests.cfg.sample
> index e01406e..cdb7b0a 100644
> --- a/client/tests/kvm/tests.cfg.sample
> +++ b/client/tests/kvm/tests.cfg.sample
> @@ -73,6 +73,14 @@ variants:
>          only Fedora.13.64
>          only unattended_install.cdrom boot shutdown
>  
> +# You may provide information about the DTM server for WHQL tests here:
> +#whql:
> +#    server_address = 10.20.30.40
> +#    server_shell_port = 10022
> +#    server_file_transfer_port = 10023
> +# Note that the DTM server must run rss.exe (available under deps/),
> +# preferably with administrator privileges.
> +
>  # Uncomment the following lines to enable abort-on-error mode:
>  #abort_on_error = yes
>  #kill_vm.* ?= no
> diff --git a/client/tests/kvm/tests_base.cfg.sample b/client/tests/kvm/tests_base.cfg.sample
> index d0b8acb..f631184 100644
> --- a/client/tests/kvm/tests_base.cfg.sample
> +++ b/client/tests/kvm/tests_base.cfg.sample
> @@ -285,6 +285,90 @@ variants:
>          iozone_cmd = "D:\IOzone\iozone.exe -a"
>          iozone_timeout = 3600
>  
> +    - @whql:         install setup unattended_install.cdrom
> +        nic_mode = tap
> +        # Replace this with the address of an installed DTM server
> +        server_address = 10.20.30.40
> +        # The server should run rss.exe like a regular Windows VM, preferably
> +        # with administrator privileges (or at least with permission to write
> +        # to the DTM studio directory)
> +        server_shell_port = 10022
> +        server_file_transfer_port = 10023
> +        server_studio_path = %programfiles%\Microsoft Driver Test Manager\Studio
> +        wtt_services = wttsvc
> +        variants:
> +            - whql_client_install:
> +                type = whql_client_install
> +                dsso_delete_machine_binary = deps/whql_delete_machine_15.exe
> +                # The username and password are required for accessing the DTM client
> +                # installer binary shared by the server
> +                server_username = administrator
> +                server_password = 1q2w3eP
> +                # This path refers to a shared directory on the server
> +                # (the final cmd will be something like \\servername\DTMInstall\...)
> +                install_cmd = \DTMInstall\Client\Setup.exe /passive
> +                install_timeout = 3600
> +            - whql_submission:    whql_client_install
> +                type = whql_submission
> +                extra_params += " -snapshot"
> +                dsso_test_binary = deps/whql_submission_15.exe
> +                test_timeout = 3600
> +                device_data = cat0 cat1 cat2 cat3 logoarch logoos whqlos whqlqual prog desc filter virt
> +                descriptors = desc1 desc2 desc3
> +                # DeviceData names
> +                dd_name_cat0     = Category
> +                dd_name_cat1     = Category
> +                dd_name_cat2     = Category
> +                dd_name_cat3     = Category
> +                dd_name_logoarch = LogoProcessorArchitecture
> +                dd_name_logoos   = LogoOperatingSystem
> +                dd_name_whqlos   = WhqlOs
> +                dd_name_whqlqual = WhqlQualification
> +                dd_name_prog     = LogoProgramId
> +                dd_name_desc     = LogoProgramDescription
> +                dd_name_filter   = WDKFilterAttribute
> +                dd_name_virt     = ParaVirtualizationDriver
> +                # Common DeviceData data
> +                dd_data_filter   = FilterIfNoInf
> +                dd_data_virt     = True
> +                variants:
> +                    - keyboard:
> +                        # test_device is a regular expression that should match a device's
> +                        # name as it appears in device manager.  The first device that matches
> +                        # is used.
> +                        test_device = keyboard
> +                        # Set timeout to 10 hours
> +                        test_timeout = 36000
> +                        dd_data_cat0 = Input\Keyboard
> +                        dd_data_cat1 = Device Fundamentals
> +                        dd_data_cat2 = System Fundamentals\Dynamic Partitioning
> +                        dd_data_prog = InputKbd
> +                        dd_data_desc = Input > Keyboard


Hello Lucas and Michael,

We have tested whql related patches in our localhost, it seems the env preparation,
tasks allocation and result collection are all good.

As you know, whql tests always take too much time, so we need to classify all the tests,
and add some variants in configuration. Do you have some plan about add other tests ?
or reserve this work to user :) ?

  Which tests could not be executed ( manual tests / not related with kvm, virtualization, ...) ?
  Which tests are basic function test ?
  Which tests take more than $n hours ?
  ....

How about split the whql configuration to a singel file, whql_test.cfg.sample ?

Regards,
Amos

> +                    - hdd:
> +                        test_device = qemu harddisk
> +                        device_data += " ex0 ex1 ex2 ex3"
> +                        dd_data_cat0 = Storage\Device Class\Disk\Disk
> +                        dd_data_cat1 = Storage\Device Class\Disk\Fixed
> +                        dd_data_cat2 = Storage\Device Class\Disk\Bus\ATA
> +                        dd_data_cat3 = Device Fundamentals
> +                        dd_data_prog = StorHDD
> +                        dd_data_desc = Storage > Hard Disk Drive (HDD)
> +                        dd_name_ex0 = Storage_bus_type
> +                        dd_data_ex0 = ATA/ATAPI
> +                        dd_name_ex1 = Hybrid_HDD_Support
> +                        dd_data_ex1 = 0
> +                        dd_name_ex2 = Non_Rotating_Media
> +                        dd_data_ex2 = 0
> +                        dd_name_ex3 = Secure_Storage
> +                        dd_data_ex3 = 0
> +                        variants:
> +                            - full:
> +                                # Yes, 100 hours, this is not a mistake
> +                                test_timeout = 360000
> +                            - syscache_test:
> +                                job_filter = syscache test
> +                                test_timeout = 7200
> +

^ permalink raw reply	[flat|nested] 8+ messages in thread

end of thread, other threads:[~2010-08-11  3:08 UTC | newest]

Thread overview: 8+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2010-07-22 10:14 [KVM-AUTOTEST PATCH v2 1/5] [RFC] KVM test: DTM automation program for WHQL tests Michael Goldish
2010-07-22 10:14 ` [KVM-AUTOTEST PATCH v2 2/5] [RFC] KVM test: DTM machine deletion tool " Michael Goldish
2010-07-22 10:14   ` [KVM-AUTOTEST PATCH v2 3/5] [RFC] KVM test: add whql_submission test Michael Goldish
2010-07-22 10:14     ` [KVM-AUTOTEST PATCH v2 4/5] [RFC] KVM test: add whql_client_install test Michael Goldish
2010-07-22 10:14       ` [KVM-AUTOTEST PATCH v2 5/5] [RFC] KVM test: add WHQL test definitions to tests_base.cfg.sample Michael Goldish
2010-08-11  3:08         ` [Autotest] " Amos Kong
2010-08-05  0:49     ` [Autotest] [KVM-AUTOTEST PATCH v2 3/5] [RFC] KVM test: add whql_submission test Amos Kong
2010-08-05 11:34       ` Michael Goldish

This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.