#!/usr/bin/env python3 # # Copyright © 2020 Google LLC # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the "Software"), # to deal in the Software without restriction, including without limitation # the rights to use, copy, modify, merge, publish, distribute, sublicense, # and/or sell copies of the Software, and to permit persons to whom the # Software is furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice (including the next # paragraph) shall be included in all copies or substantial portions of the # Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. import argparse import queue import re from serial_buffer import SerialBuffer import sys import threading class CrosServoRun: def __init__(self, cpu, ec): # Merged FIFO for the two serial buffers, fed by threads. self.serial_queue = queue.Queue() self.sentinel = object() self.threads_done = 0 self.ec_ser = SerialBuffer( ec, "results/serial-ec.txt", "R SERIAL-EC> ") self.cpu_ser = SerialBuffer( cpu, "results/serial.txt", "R SERIAL-CPU> ") self.iter_feed_ec = threading.Thread( target=self.iter_feed_queue, daemon=True, args=(self.ec_ser.lines(),)) self.iter_feed_ec.start() self.iter_feed_cpu = threading.Thread( target=self.iter_feed_queue, daemon=True, args=(self.cpu_ser.lines(),)) self.iter_feed_cpu.start() # Feed lines from our serial queues into the merged queue, marking when our # input is done. def iter_feed_queue(self, it): for i in it: self.serial_queue.put(i) self.serial_queue.put(sentinel) # Return the next line from the queue, counting how many threads have # terminated and joining when done def get_serial_queue_line(self): line = self.serial_queue.get() if line == self.sentinel: self.threads_done = self.threads_done + 1 if self.threads_done == 2: self.iter_feed_cpu.join() self.iter_feed_ec.join() return line # Returns an iterator for getting the next line. def serial_queue_lines(self): return iter(self.get_serial_queue_line, self.sentinel) def ec_write(self, s): print("W SERIAL-EC> %s" % s) self.ec_ser.serial.write(s.encode()) def cpu_write(self, s): print("W SERIAL-CPU> %s" % s) self.cpu_ser.serial.write(s.encode()) def print_error(self, message): RED = '\033[0;31m' NO_COLOR = '\033[0m' print(RED + message + NO_COLOR) def run(self): # Flush any partial commands in the EC's prompt, then ask for a reboot. self.ec_write("\n") self.ec_write("reboot\n") # This is emitted right when the bootloader pauses to check for input. # Emit a ^N character to request network boot, because we don't have a # direct-to-netboot firmware on cheza. for line in self.serial_queue_lines(): if re.search("load_archive: loading locale_en.bin", line): self.cpu_write("\016") break # The Cheza boards have issues with failing to bring up power to # the system sometimes, possibly dependent on ambient temperature # in the farm. if re.search("POWER_GOOD not seen in time", line): self.print_error("Detected intermittent poweron failure, restarting run...") return 2 tftp_failures = 0 for line in self.serial_queue_lines(): if re.search("---. end Kernel panic", line): return 1 # The Cheza firmware seems to occasionally get stuck looping in # this error state during TFTP booting, possibly based on amount of # network traffic around it, but it'll usually recover after a # reboot. if re.search("R8152: Bulk read error 0xffffffbf", line): tftp_failures += 1 if tftp_failures >= 100: self.print_error("Detected intermittent tftp failure, restarting run...") return 2 # There are very infrequent bus errors during power management transitions # on cheza, which we don't expect to be the case on future boards. if re.search("Kernel panic - not syncing: Asynchronous SError Interrupt", line): self.print_error("Detected cheza power management bus error, restarting run...") return 2 # If the network device dies, it's probably not graphics's fault, just try again. if re.search("NETDEV WATCHDOG", line): self.print_error( "Detected network device failure, restarting run...") return 2 # These HFI response errors started appearing with the introduction # of piglit runs. CosmicPenguin says: # # "message ID 106 isn't a thing, so likely what happened is that we # got confused when parsing the HFI queue. If it happened on only # one run, then memory corruption could be a possible clue" # # Given that it seems to trigger randomly near a GPU fault and then # break many tests after that, just restart the whole run. if re.search("a6xx_hfi_send_msg.*Unexpected message id .* on the response queue", line): self.print_error("Detected cheza power management bus error, restarting run...") return 2 if re.search("coreboot.*bootblock starting", line): self.print_error( "Detected spontaneous reboot, restarting run...") return 2 result = re.search("hwci: mesa: (\S*)", line) if result: if result.group(1) == "pass": return 0 else: return 1 self.print_error("Reached the end of the CPU serial log without finding a result") return 1 def main(): parser = argparse.ArgumentParser() parser.add_argument('--cpu', type=str, help='CPU Serial device', required=True) parser.add_argument( '--ec', type=str, help='EC Serial device', required=True) args = parser.parse_args() servo = CrosServoRun(args.cpu, args.ec) while True: retval = servo.run() if retval != 2: break # power down the CPU on the device servo.ec_write("power off\n") sys.exit(retval) if __name__ == '__main__': main()