Software | Ritchie Lee

AdaptiveStressTesting.jl

AdaptiveStressTesting.jl is a software package written in Julia for stress testing safety-critical systems. The package uses reinforcement learning to search for the highest-probability path from an initially healthy system state to a failure event. The package has been applied to find scenarios of near mid-air collisions (NMACs) in prototypes of the Next-Generation Airborne Collision Avoidance System (ACAS X).

Related Publications:

R. Lee and L. Shen, System identification of a Cessna 182 model UAV, 2007.
[Bibtex]

@Unpublished{Lee2007,
author = {Ritchie Lee and Lu Shen},
title = {System Identification of a {C}essna 182 Model {UAV}},
organization = {{S}tanford {U}niversity CS229},
year = {2007}
}

R. Lee and C. Ippolito, “A perception and mapping approach for plume detection in payload directed flight,” in Aiaa [email protected] conference, 2009.
[Bibtex]

@InProceedings{Lee2009,
author = {Lee, Ritchie and Ippolito, Corey},
title = {A Perception and Mapping Approach for Plume Detection in Payload Directed Flight},
booktitle = aiaa_info,
organization = {AIAA},
year = {2009},
month = {April}
}

R. Lee, C. Ippolito, Y. Yeh, J. Spritzer, and G. Phelps, “Payload-directed control of geophysical magnetic surveys,” in Aiaa [email protected] conference, 2010.
[Bibtex]

@InProceedings{Lee2010a,
Title = {Payload-Directed Control of Geophysical Magnetic Surveys},
Author = {Lee, Ritchie and Ippolito, Corey and Yeh, Yoo-Hsiu and Spritzer, John and Phelps, Geoffrey},
Booktitle = aiaa_info,
organization = {AIAA},
Year = {2010},
Month = {April}
}

R. Lee and D. H. Wolpert, “Game theoretic modeling of human behavior in mid-air encounters,” in NIPS workshop on decision making with multiple imperfect decision makers, 2010.
[Bibtex]

@InProceedings{Lee2010b,
Title = {Game Theoretic Modeling of Human Behavior in Mid-Air Encounters},
Author = {Lee, Ritchie and Wolpert, David H},
Booktitle = {{NIPS} Workshop on Decision Making with Multiple Imperfect Decision Makers},
Year = {2010}
}

J. Dolan, R. Lee, Y. Yeh, C. Yeh, D. Y. Nguyen, S. Ben-Menahem, and A. K. Ishihara, “Neural network estimation of photovoltaic I–V curves under partially shaded conditions,” in International joint conference on neural networks (ijcnn), 2011, p. 1358–1365.
[Bibtex]

@InProceedings{Dolan2011,
author = {Dolan, Jacques and Lee, Ritchie and Yeh, Yoo-Hsiu and Yeh, Chiping and Nguyen, Daniel Y and Ben-Menahem, Shahar and Ishihara, Abraham K},
title = {Neural Network Estimation of Photovoltaic {I}--{V} Curves Under Partially Shaded Conditions},
booktitle = ijcnn,
year = {2011},
pages = {1358--1365},
organization = {IEEE}
}

R. Lee, S. Backhaus, J. Bono, D. H. Wolpert, R. Bent, and B. Tracey, “Modeling humans as reinforcement learners: how to predict human behavior in multi-stage games,” in NIPS workshop on decision making with multiple imperfect decision makers, 2011.
[Bibtex]

@InProceedings{Lee2011,
author = {Lee, Ritchie and Backhaus, Scott and Bono, James and Wolpert, David H. and Bent, Russell and Tracey, Brendan},
title = {Modeling Humans as Reinforcement Learners: How to Predict Human Behavior in Multi-Stage Games},
booktitle = {{NIPS} Workshop on Decision Making with Multiple Imperfect Decision Makers},
year = {2011}
}

R. Lee and D. H. Wolpert, “Game theoretic modeling of pilot behavior during mid-air encounters,” in Decision making with imperfect decision makers, Springer, 2012, vol. 28, pp. 75-111.
[Bibtex]

@InCollection{Lee2012,
author = {Lee, Ritchie and Wolpert, David H.},
title = {Game Theoretic Modeling of Pilot Behavior During Mid-Air Encounters},
booktitle = {Decision Making with Imperfect Decision Makers},
publisher = {Springer},
year = {2012},
volume = {28},
series = {Intelligent Systems Reference Library},
chapter = {4},
pages = {75-111}
}

E. Denney, C. Ippolito, R. Lee, and G. Pai, “An integrated safety and systems engineering methodology for small unmanned aircraft systems,” in Aiaa [email protected] conference, 2012.
[Bibtex]

@InProceedings{Denney2012,
Title = {An Integrated Safety and Systems Engineering Methodology for Small Unmanned Aircraft Systems},
Author = {Ewen Denney and Corey Ippolito and Ritchie Lee and Ganesh Pai},
Booktitle = aiaa_info,
Organization = {AIAA},
Year = {2012}
}

E. J. Schlicht, R. Lee, D. H. Wolpert, M. J. Kochenderfer, and B. Tracey, “Predicting the behavior of interacting humans by fusing data from multiple sources,” in Conference on uncertainty in artificial intelligence (uai), 2012.
[Bibtex]

@InProceedings{Schlicht2012,
Title = {Predicting the Behavior of Interacting Humans by Fusing Data from Multiple Sources},
Author = {Erik J. Schlicht and Ritchie Lee and David H. Wolpert and Mykel J. Kochenderfer and Brendan Tracey},
Booktitle = uai,
Organization = {AUAI},
Year = {2012}
}

G. Yan, R. Lee, A. Kent, and D. H. Wolpert, “Towards a Bayesian network game framework for evaluating DDoS attacks and defense,” in Conference on computer and communications security (ccs), 2012, p. 553–566.
[Bibtex]

@InProceedings{Yan2012,
Title = {Towards a {B}ayesian Network Game Framework for Evaluating {DDoS} Attacks and Defense},
Author = {Yan, Guanhua and Lee, Ritchie and Kent, Alex and Wolpert, David H.},
Booktitle = acmccs,
Year = {2012},
Organization = {ACM},
Pages = {553--566}
}

Y. Yildiz, R. Lee, and G. Brat, “Using game theoretic models to predict pilot behavior in NextGen merging and landing scenario,” in Aiaa scitech, modeling and simulation technologies conference (mst), 2012.
[Bibtex]

@InProceedings{Yildiz2012,
Title = {Using Game Theoretic Models to Predict Pilot Behavior in {N}ext{G}en Merging and Landing Scenario},
Author = {Yildiz, Yildiray and Lee, Ritchie and Brat, Guillaume},
Booktitle = aiaa_mst,
Organization = {AIAA},
Year = {2012}
}

R. Lee, D. H. Wolpert, J. Bono, S. Backhaus, R. Bent, and B. Tracey, “Counter-factual reinforcement learning: how to model decision-makers that anticipate the future,” in Decision making and imperfection, Springer, 2013, vol. 474, pp. 101-128.
[Bibtex]

@InCollection{Lee2013,
author = {Lee, Ritchie and Wolpert, David H. and Bono, James and Backhaus, Scott and Bent, Russell and Tracey, Brendan},
title = {Counter-Factual Reinforcement Learning: How to Model Decision-Makers That Anticipate the Future},
booktitle = {Decision Making and Imperfection},
publisher = {Springer},
year = {2013},
volume = {474},
series = {Studies in Computational Intelligence},
chapter = {4},
pages = {101-128}
}

S. Backhaus, R. Bent, J. Bono, R. Lee, B. Tracey, D. H. Wolpert, D. Xie, and Y. Yildiz, “Cyber-physical security: a game theory model of humans interacting over control systems,” Ieee transactions on smart grid, vol. 4, iss. 4, p. 2320–2327, 2013.
[Bibtex]

@Article{Backhaus2013,
author = {Backhaus, Scott and Bent, Russell and Bono, James and Lee, Ritchie and Tracey, Brendan and Wolpert, David H. and Xie, Dongping and Yildiz, Yildiray},
title = {Cyber-Physical Security: A Game Theory Model of Humans Interacting Over Control Systems},
journal = ieeesg,
year = {2013},
volume = {4},
number = {4},
pages = {2320--2327},
publisher = {IEEE}
}

R. Lee and M. Rajabi, Assessing NuPIC and CLA in a machine learning context using NASA aviation datasets, 2014.
[Bibtex]

@Unpublished{Lee2014,
author = {Ritchie Lee and Mariam Rajabi},
title = {Assessing {NuPIC} and {CLA} in a Machine Learning Context Using {NASA} Aviation Datasets},
organization = {Carnegie Mellon University EE18799-SV},
year = {2014}
}

G. A. Phelps, C. Ippolito, R. Lee, J. Spritzer, and Y. Yeh, “Investigations into near-real-time surveying for geophysical data collection using an autonomous ground vehicle,” United States Geological Survey (USGS) 2014.
[Bibtex]

@TechReport{Phelps2014,
author = {Phelps, Geoffrey A and Ippolito, Corey and Lee, Ritchie and Spritzer, John and Yeh, Yoo-Hsiu},
title = {Investigations into Near-Real-Time Surveying for Geophysical Data Collection Using an Autonomous Ground Vehicle},
institution = usgs,
year = {2014}
}

R. Lee, M. J. Kochenderfer, O. J. Mengshoel, G. P. Brat, and M. P. Owen, “Adaptive stress testing of airborne collision avoidance systems,” in Ieee/aiaa digital avionics systems conference (dasc), 2015.
[Bibtex]

@InProceedings{Lee2015,
author = {Ritchie Lee and Mykel J. Kochenderfer and Ole J. Mengshoel and Guillaume P. Brat and Michael P. Owen},
title = {Adaptive Stress Testing of Airborne Collision Avoidance Systems},
booktitle = dasc,
organization = {AIAA/IEEE},
note = {Best Paper of Session},
year = {2015}
}

A. Agogino, A. Iscen, R. Lee, D. Bowers, K. Tumer, and G. Brat, “Scalable hierarchical multifidelity simulation and multiagent optimization of air traffic,” in AAMAS workshop on massive multiagent systems, 2015.
[Bibtex]

@InProceedings{Agogino2015,
Title = {Scalable Hierarchical Multifidelity Simulation and Multiagent Optimization of Air Traffic},
Author = {Adrian Agogino and Atil Iscen and Ritchie Lee and Devin Bowers and Kagan Tumer and Guillaume Brat},
Booktitle = {{AAMAS} Workshop on Massive Multiagent Systems},
Year = {2015}
}

R. Lee, A. Agogino, and D. Giannakopoulou, “Algorithms and tools for model-based V&V of autonomy software,” NASA 2017.
[Bibtex]

@TechReport{Lee2017,
Title = {Algorithms and Tools for Model-Based {V\&V} of Autonomy Software},
Author = {Lee, Ritchie and Agogino, Adrian and Giannakopoulou, Dimitra},
Institution = {NASA},
Year = {2017}
}

R. Lee, O. J. Mengshoel, A. Saksena, R. Gardner, D. Genin, J. Brush, and M. J. Kochenderfer, “Differential adaptive stress testing of airborne collision avoidance systems,” in Aiaa scitech, modeling and simulation technologies conference (mst), 2018.
[Bibtex]

@InProceedings{Lee2018a,
author = {Lee, Ritchie and Mengshoel, Ole J. and Saksena, Anshu and Gardner, Ryan and Genin, Daniel and Brush, Jeffrey and Kochenderfer, Mykel J.},
title = {Differential Adaptive Stress Testing of Airborne Collision Avoidance Systems},
booktitle = aiaa_mst,
year = {2018},
note = {Best Paper Award},
Organization = {AIAA}
}

R. Lee, M. J. Kochenderfer, O. J. Mengshoel, and J. Silbermann, “Interpretable categorization of heterogeneous time series data,” in International conference on data mining (sdm), 2018.
[Bibtex]

@InProceedings{Lee2018b,
author = {Lee, Ritchie and Kochenderfer, Mykel J. and Mengshoel, Ole J. and Silbermann, Joshua},
title = {Interpretable Categorization of Heterogeneous Time Series Data},
booktitle = sdm,
year = {2018},
Organization = {SIAM}
}

M. Koren, S. Alsaif, R. Lee, and M. J. Kochenderfer, “Adaptive stress testing for autonomous vehicles,” in Ieee intelligent vehicles symposium (iv), 2018.
[Bibtex]

@InProceedings{Koren2018,
author = {Mark Koren and Saud Alsaif and Ritchie Lee and Mykel J. Kochenderfer},
title = {Adaptive Stress Testing for Autonomous Vehicles},
booktitle = iv,
Organization = {IEEE},
year = {2018}
}

R. Lee, O. J. Mengshoel, A. K. Agogino, D. Giannakopoulou, and M. J. Kochenderfer, “Adaptive stress testing of trajectory planning systems,” in Aiaa scitech, intelligent systems conference (is), 2019.
[Bibtex]

@inproceedings{Lee2019a,
author = {Lee, Ritchie and Mengshoel, Ole J. and Agogino, Adrian K. and Giannakopoulou, Dimitra and Kochenderfer, Mykel J.},
title = {Adaptive Stress Testing of Trajectory Planning Systems},
booktitle = aiaa_is,
year = {2019},
organization = {AIAA}
}

A. K. Agogino, R. Lee, and D. Giannakopoulou, “Challenges of explaining real-time planning,” in ICAPS workshop on explainable planning (XAIP), 2019.
[Bibtex]

@InProceedings{Agogino2019a,
Author = {Agogino, Adrian K. and Lee, Ritchie and Giannakopoulou, Dimitra},
title = {Challenges of Explaining Real-Time Planning},
booktitle = {{ICAPS} Workshop on Explainable Planning ({XAIP})},
year = {2019}
}

R. Lee, “Adastress: adaptive stress testing and interpretable categorization for safety-critical systems,” PhD Thesis, 2019.
[Bibtex]

@phdthesis{leethesis,
author = {Ritchie Lee},
title = {AdaStress: Adaptive Stress Testing and Interpretable Categorization for Safety-Critical Systems},
school = {Carnegie Mellon University},
year = {2019},
month = {May}
}

R. Lee, O. J. Mengshoel, and M. J. Kochenderfer, “Adaptive stress testing of safety-critical systems,” in Safe, autonomous, and intelligent vehicles, Springer, 2019, p. 77–95.
[Bibtex]

@InCollection{Lee2019b,
author = {Lee, Ritchie and Mengshoel, Ole J. and Kochenderfer, Mykel J.},
title = {Adaptive Stress Testing of Safety-Critical Systems},
booktitle = {Safe, Autonomous, and Intelligent Vehicles},
series = {Unmanned Systems Technology},
publisher = {Springer},
year = {2019},
pages = {77--95}
}

R. Lee, Clarke Justin, A. K. Agogino, and D. Giannakopoulou, “Improving trust in deep neural networks using nearest neighbors,” in Aiaa scitech, intelligent systems conference (is), 2020.
[Bibtex]

@inproceedings{Lee2020a,
author = {Lee, Ritchie and Clarke, Justin, and Agogino, Adrian K. and Giannakopoulou, Dimitra},
title = {Improving Trust in Deep Neural Networks using Nearest Neighbors},
booktitle = aiaa_is,
year = {2020},
organization = {AIAA}
}

R. J. Moss, R. Lee, and M. J. Kochenderfer, “Adaptive stress testing of trajectory predictions in flight management systems,” in Ieee/aiaa digital avionics systems conference (dasc), 2020.
[Bibtex]

@InProceedings{Moss2020a,
author = {Robert J. Moss and Ritchie Lee and Mykel J. Kochenderfer},
title = {Adaptive Stress Testing of Trajectory Predictions in Flight Management Systems},
booktitle = dasc,
organization = {AIAA/IEEE},
note = {Best Paper of Session},
year = {2020}
}

A. Corso, R. Lee, and M. J. Kochenderfer, “Scalable autonomous vehicle safety validation through dynamic programming and scene decomposition,” in Ieee international conference on intelligent transportation systems (itsc), 2020.
[Bibtex]
```
@InProceedings{Corso2020a,
author = {Anthony Corso and Ritchie Lee and Mykel J. Kochenderfer},
booktitle = itsc,
title = {Scalable Autonomous Vehicle Safety Validation Through Dynamic Programming and Scene Decomposition},
year = {2020}
}
```
K. D. Julian, R. Lee, and M. J. Kochenderfer, “Validation of image-based neural network controllers through adaptive stress testing,” in Ieee international conference on intelligent transportation systems (itsc), 2020.
[Bibtex]
```
@InProceedings{Julian2020a,
author = {Kyle D. Julian and Ritchie Lee and Mykel J. Kochenderfer},
booktitle = itsc,
title = {Validation of Image-Based Neural Network Controllers Through Adaptive Stress Testing},
year = {2020}
}
```

R. Lee, O. J. Mengshoel, A. Saksena, R. Gardner, D. Genin, J. Silbermann, M. Owen, and M. J. Kochenderfer, “Adaptive stress testing: finding likely failure events with reinforcement learning,” Journal of artificial intelligence research, vol. 69, p. 1165–1201, 2020.
[Bibtex]

@article{Lee2020b,
author = {Lee, Ritchie and Mengshoel, Ole J. and Saksena, Anshu and Gardner, Ryan and Genin, Daniel and Silbermann, Joshua and Owen, Michael and Kochenderfer, Mykel J.},
title = {Adaptive Stress Testing: Finding Likely Failure Events with Reinforcement Learning},
journal = jair,
year = {2020},
volume = {69},
pages = {1165--1201}
}

Proc. 12th international NASA formal methods symposium, R. Lee, S. Jha, A. Mavridou, and D. Giannakoupoulou, Eds., Springer, 2020, vol. 12229.
[Bibtex]

@book{Lee2020c,
title = {Proc. 12th International {NASA} Formal Methods Symposium},
editor = {Ritchie Lee and Susmit Jha and Anastasia Mavridou and Dimitra Giannakoupoulou},
series = {Lecture Notes in Computer Science},
volume = {12229},
publisher = {Springer},
year = {2020},
}

A. K. Agogino, R. Lee, and D. Giannakopoulou, “Machine learning explainability and transferability for path navigation,” in Aiaa scitech, intelligent systems conference (is), 2021.
[Bibtex]

@inproceedings{Agogino2021a,
author = {Agogino, Adrian K. and Lee, Ritchie and Giannakopoulou, Dimitra},
title = {Machine Learning Explainability and Transferability for Path Navigation},
booktitle = aiaa_is,
year = {2021},
organization = {AIAA}
}

A. Corso, R. J. Moss, M. Koren, R. Lee, and M. J. Kochenderfer, “A survey of algorithms for black-box safety validation,” Journal of artificial intelligence research, vol. 72, p. 377–428, 2021.
[Bibtex]

@Article{Corso2021a,
author = {Anthony Corso and Robert J. Moss and Mark Koren and Ritchie Lee and Mykel J. Kochenderfer},
title = {A Survey of Algorithms for Black-box Safety Validation},
journal = jair,
year = {2021},
volume = {72},
pages = {377--428}
}

M. Durling, H. Herencia-Zapana, B. Meng, M. Meiners, J. Hochwarth, N. Visser, R. Lee, R. Moss, and V. T. Valapil, “Certification considerations for adaptive stress testing of airborne software,” in Ieee/aiaa digital avionics systems conference (dasc), 2021.
[Bibtex]

@InProceedings{Durling2021a,
title = {Certification Considerations for Adaptive Stress Testing of Airborne Software},
author = {Durling, Michael and Herencia-Zapana, Heber and Meng, Baoluo and Meiners, Mike and Hochwarth, Joachim and Visser, Nicholas and Lee, Ritchie and Moss, Robert and Valapil, Vidhya Tekken},
booktitle = dasc,
year = {2021},
organization = {AIAA/IEEE}
}

R. Lipkis, R. Lee, J. Silbermann, and T. Young, “Adaptive stress testing of collision avoidance systems for small uass with deep reinforcement learning,” in Aiaa scitech, intelligent systems conference (is), 2022.
[Bibtex]

@inproceedings{Lipkis2022,
title = {Adaptive Stress Testing of Collision Avoidance Systems for Small UASs with Deep Reinforcement Learning},
author = {Lipkis, Rory and Lee, Ritchie and Silbermann, Joshua and Young, Tyler},
booktitle = aiaa_is,
organization = {AIAA},
year = {2022}
}

GBDTs.jl

GBDTs.jl is a software package written in Julia for learning Grammar-Based Decision Trees (GBDTs). GBDT combines decision trees, context-free grammars, and expression optimization for automatically categorizing and explaining data. The package has been applied to categorize scenarios of near mid-air collisions (NMACs) in prototypes of the Next-Generation Airborne Collision Avoidance System (ACAS X).

Related Publications:

R. Lee and L. Shen, System identification of a Cessna 182 model UAV, 2007.
[Bibtex]

@Unpublished{Lee2007,
author = {Ritchie Lee and Lu Shen},
title = {System Identification of a {C}essna 182 Model {UAV}},
organization = {{S}tanford {U}niversity CS229},
year = {2007}
}

R. Lee and C. Ippolito, “A perception and mapping approach for plume detection in payload directed flight,” in Aiaa [email protected] conference, 2009.
[Bibtex]

@InProceedings{Lee2009,
author = {Lee, Ritchie and Ippolito, Corey},
title = {A Perception and Mapping Approach for Plume Detection in Payload Directed Flight},
booktitle = aiaa_info,
organization = {AIAA},
year = {2009},
month = {April}
}

R. Lee, C. Ippolito, Y. Yeh, J. Spritzer, and G. Phelps, “Payload-directed control of geophysical magnetic surveys,” in Aiaa [email protected] conference, 2010.
[Bibtex]

@InProceedings{Lee2010a,
Title = {Payload-Directed Control of Geophysical Magnetic Surveys},
Author = {Lee, Ritchie and Ippolito, Corey and Yeh, Yoo-Hsiu and Spritzer, John and Phelps, Geoffrey},
Booktitle = aiaa_info,
organization = {AIAA},
Year = {2010},
Month = {April}
}

R. Lee and D. H. Wolpert, “Game theoretic modeling of human behavior in mid-air encounters,” in NIPS workshop on decision making with multiple imperfect decision makers, 2010.
[Bibtex]

@InProceedings{Lee2010b,
Title = {Game Theoretic Modeling of Human Behavior in Mid-Air Encounters},
Author = {Lee, Ritchie and Wolpert, David H},
Booktitle = {{NIPS} Workshop on Decision Making with Multiple Imperfect Decision Makers},
Year = {2010}
}

@InProceedings{Dolan2011,
author = {Dolan, Jacques and Lee, Ritchie and Yeh, Yoo-Hsiu and Yeh, Chiping and Nguyen, Daniel Y and Ben-Menahem, Shahar and Ishihara, Abraham K},
title = {Neural Network Estimation of Photovoltaic {I}--{V} Curves Under Partially Shaded Conditions},
booktitle = ijcnn,
year = {2011},
pages = {1358--1365},
organization = {IEEE}
}

@InProceedings{Lee2011,
author = {Lee, Ritchie and Backhaus, Scott and Bono, James and Wolpert, David H. and Bent, Russell and Tracey, Brendan},
title = {Modeling Humans as Reinforcement Learners: How to Predict Human Behavior in Multi-Stage Games},
booktitle = {{NIPS} Workshop on Decision Making with Multiple Imperfect Decision Makers},
year = {2011}
}

R. Lee and D. H. Wolpert, “Game theoretic modeling of pilot behavior during mid-air encounters,” in Decision making with imperfect decision makers, Springer, 2012, vol. 28, pp. 75-111.
[Bibtex]

@InCollection{Lee2012,
author = {Lee, Ritchie and Wolpert, David H.},
title = {Game Theoretic Modeling of Pilot Behavior During Mid-Air Encounters},
booktitle = {Decision Making with Imperfect Decision Makers},
publisher = {Springer},
year = {2012},
volume = {28},
series = {Intelligent Systems Reference Library},
chapter = {4},
pages = {75-111}
}

E. Denney, C. Ippolito, R. Lee, and G. Pai, “An integrated safety and systems engineering methodology for small unmanned aircraft systems,” in Aiaa [email protected] conference, 2012.
[Bibtex]

@InProceedings{Denney2012,
Title = {An Integrated Safety and Systems Engineering Methodology for Small Unmanned Aircraft Systems},
Author = {Ewen Denney and Corey Ippolito and Ritchie Lee and Ganesh Pai},
Booktitle = aiaa_info,
Organization = {AIAA},
Year = {2012}
}

@InProceedings{Schlicht2012,
Title = {Predicting the Behavior of Interacting Humans by Fusing Data from Multiple Sources},
Author = {Erik J. Schlicht and Ritchie Lee and David H. Wolpert and Mykel J. Kochenderfer and Brendan Tracey},
Booktitle = uai,
Organization = {AUAI},
Year = {2012}
}

@InProceedings{Yan2012,
Title = {Towards a {B}ayesian Network Game Framework for Evaluating {DDoS} Attacks and Defense},
Author = {Yan, Guanhua and Lee, Ritchie and Kent, Alex and Wolpert, David H.},
Booktitle = acmccs,
Year = {2012},
Organization = {ACM},
Pages = {553--566}
}

@InProceedings{Yildiz2012,
Title = {Using Game Theoretic Models to Predict Pilot Behavior in {N}ext{G}en Merging and Landing Scenario},
Author = {Yildiz, Yildiray and Lee, Ritchie and Brat, Guillaume},
Booktitle = aiaa_mst,
Organization = {AIAA},
Year = {2012}
}

@InCollection{Lee2013,
author = {Lee, Ritchie and Wolpert, David H. and Bono, James and Backhaus, Scott and Bent, Russell and Tracey, Brendan},
title = {Counter-Factual Reinforcement Learning: How to Model Decision-Makers That Anticipate the Future},
booktitle = {Decision Making and Imperfection},
publisher = {Springer},
year = {2013},
volume = {474},
series = {Studies in Computational Intelligence},
chapter = {4},
pages = {101-128}
}

@Article{Backhaus2013,
author = {Backhaus, Scott and Bent, Russell and Bono, James and Lee, Ritchie and Tracey, Brendan and Wolpert, David H. and Xie, Dongping and Yildiz, Yildiray},
title = {Cyber-Physical Security: A Game Theory Model of Humans Interacting Over Control Systems},
journal = ieeesg,
year = {2013},
volume = {4},
number = {4},
pages = {2320--2327},
publisher = {IEEE}
}

R. Lee and M. Rajabi, Assessing NuPIC and CLA in a machine learning context using NASA aviation datasets, 2014.
[Bibtex]

@Unpublished{Lee2014,
author = {Ritchie Lee and Mariam Rajabi},
title = {Assessing {NuPIC} and {CLA} in a Machine Learning Context Using {NASA} Aviation Datasets},
organization = {Carnegie Mellon University EE18799-SV},
year = {2014}
}

@TechReport{Phelps2014,
author = {Phelps, Geoffrey A and Ippolito, Corey and Lee, Ritchie and Spritzer, John and Yeh, Yoo-Hsiu},
title = {Investigations into Near-Real-Time Surveying for Geophysical Data Collection Using an Autonomous Ground Vehicle},
institution = usgs,
year = {2014}
}

@InProceedings{Lee2015,
author = {Ritchie Lee and Mykel J. Kochenderfer and Ole J. Mengshoel and Guillaume P. Brat and Michael P. Owen},
title = {Adaptive Stress Testing of Airborne Collision Avoidance Systems},
booktitle = dasc,
organization = {AIAA/IEEE},
note = {Best Paper of Session},
year = {2015}
}

@InProceedings{Agogino2015,
Title = {Scalable Hierarchical Multifidelity Simulation and Multiagent Optimization of Air Traffic},
Author = {Adrian Agogino and Atil Iscen and Ritchie Lee and Devin Bowers and Kagan Tumer and Guillaume Brat},
Booktitle = {{AAMAS} Workshop on Massive Multiagent Systems},
Year = {2015}
}

R. Lee, A. Agogino, and D. Giannakopoulou, “Algorithms and tools for model-based V&V of autonomy software,” NASA 2017.
[Bibtex]

@TechReport{Lee2017,
Title = {Algorithms and Tools for Model-Based {V\&V} of Autonomy Software},
Author = {Lee, Ritchie and Agogino, Adrian and Giannakopoulou, Dimitra},
Institution = {NASA},
Year = {2017}
}

@InProceedings{Lee2018a,
author = {Lee, Ritchie and Mengshoel, Ole J. and Saksena, Anshu and Gardner, Ryan and Genin, Daniel and Brush, Jeffrey and Kochenderfer, Mykel J.},
title = {Differential Adaptive Stress Testing of Airborne Collision Avoidance Systems},
booktitle = aiaa_mst,
year = {2018},
note = {Best Paper Award},
Organization = {AIAA}
}

R. Lee, M. J. Kochenderfer, O. J. Mengshoel, and J. Silbermann, “Interpretable categorization of heterogeneous time series data,” in International conference on data mining (sdm), 2018.
[Bibtex]

@InProceedings{Lee2018b,
author = {Lee, Ritchie and Kochenderfer, Mykel J. and Mengshoel, Ole J. and Silbermann, Joshua},
title = {Interpretable Categorization of Heterogeneous Time Series Data},
booktitle = sdm,
year = {2018},
Organization = {SIAM}
}

M. Koren, S. Alsaif, R. Lee, and M. J. Kochenderfer, “Adaptive stress testing for autonomous vehicles,” in Ieee intelligent vehicles symposium (iv), 2018.
[Bibtex]

@InProceedings{Koren2018,
author = {Mark Koren and Saud Alsaif and Ritchie Lee and Mykel J. Kochenderfer},
title = {Adaptive Stress Testing for Autonomous Vehicles},
booktitle = iv,
Organization = {IEEE},
year = {2018}
}

@inproceedings{Lee2019a,
author = {Lee, Ritchie and Mengshoel, Ole J. and Agogino, Adrian K. and Giannakopoulou, Dimitra and Kochenderfer, Mykel J.},
title = {Adaptive Stress Testing of Trajectory Planning Systems},
booktitle = aiaa_is,
year = {2019},
organization = {AIAA}
}

A. K. Agogino, R. Lee, and D. Giannakopoulou, “Challenges of explaining real-time planning,” in ICAPS workshop on explainable planning (XAIP), 2019.
[Bibtex]

@InProceedings{Agogino2019a,
Author = {Agogino, Adrian K. and Lee, Ritchie and Giannakopoulou, Dimitra},
title = {Challenges of Explaining Real-Time Planning},
booktitle = {{ICAPS} Workshop on Explainable Planning ({XAIP})},
year = {2019}
}

R. Lee, “Adastress: adaptive stress testing and interpretable categorization for safety-critical systems,” PhD Thesis, 2019.
[Bibtex]

@phdthesis{leethesis,
author = {Ritchie Lee},
title = {AdaStress: Adaptive Stress Testing and Interpretable Categorization for Safety-Critical Systems},
school = {Carnegie Mellon University},
year = {2019},
month = {May}
}

R. Lee, O. J. Mengshoel, and M. J. Kochenderfer, “Adaptive stress testing of safety-critical systems,” in Safe, autonomous, and intelligent vehicles, Springer, 2019, p. 77–95.
[Bibtex]

@InCollection{Lee2019b,
author = {Lee, Ritchie and Mengshoel, Ole J. and Kochenderfer, Mykel J.},
title = {Adaptive Stress Testing of Safety-Critical Systems},
booktitle = {Safe, Autonomous, and Intelligent Vehicles},
series = {Unmanned Systems Technology},
publisher = {Springer},
year = {2019},
pages = {77--95}
}

@inproceedings{Lee2020a,
author = {Lee, Ritchie and Clarke, Justin, and Agogino, Adrian K. and Giannakopoulou, Dimitra},
title = {Improving Trust in Deep Neural Networks using Nearest Neighbors},
booktitle = aiaa_is,
year = {2020},
organization = {AIAA}
}

@InProceedings{Moss2020a,
author = {Robert J. Moss and Ritchie Lee and Mykel J. Kochenderfer},
title = {Adaptive Stress Testing of Trajectory Predictions in Flight Management Systems},
booktitle = dasc,
organization = {AIAA/IEEE},
note = {Best Paper of Session},
year = {2020}
}

A. Corso, R. Lee, and M. J. Kochenderfer, “Scalable autonomous vehicle safety validation through dynamic programming and scene decomposition,” in Ieee international conference on intelligent transportation systems (itsc), 2020.
[Bibtex]
```
@InProceedings{Corso2020a,
author = {Anthony Corso and Ritchie Lee and Mykel J. Kochenderfer},
booktitle = itsc,
title = {Scalable Autonomous Vehicle Safety Validation Through Dynamic Programming and Scene Decomposition},
year = {2020}
}
```
K. D. Julian, R. Lee, and M. J. Kochenderfer, “Validation of image-based neural network controllers through adaptive stress testing,” in Ieee international conference on intelligent transportation systems (itsc), 2020.
[Bibtex]
```
@InProceedings{Julian2020a,
author = {Kyle D. Julian and Ritchie Lee and Mykel J. Kochenderfer},
booktitle = itsc,
title = {Validation of Image-Based Neural Network Controllers Through Adaptive Stress Testing},
year = {2020}
}
```

@article{Lee2020b,
author = {Lee, Ritchie and Mengshoel, Ole J. and Saksena, Anshu and Gardner, Ryan and Genin, Daniel and Silbermann, Joshua and Owen, Michael and Kochenderfer, Mykel J.},
title = {Adaptive Stress Testing: Finding Likely Failure Events with Reinforcement Learning},
journal = jair,
year = {2020},
volume = {69},
pages = {1165--1201}
}

Proc. 12th international NASA formal methods symposium, R. Lee, S. Jha, A. Mavridou, and D. Giannakoupoulou, Eds., Springer, 2020, vol. 12229.
[Bibtex]

@book{Lee2020c,
title = {Proc. 12th International {NASA} Formal Methods Symposium},
editor = {Ritchie Lee and Susmit Jha and Anastasia Mavridou and Dimitra Giannakoupoulou},
series = {Lecture Notes in Computer Science},
volume = {12229},
publisher = {Springer},
year = {2020},
}

A. K. Agogino, R. Lee, and D. Giannakopoulou, “Machine learning explainability and transferability for path navigation,” in Aiaa scitech, intelligent systems conference (is), 2021.
[Bibtex]

@inproceedings{Agogino2021a,
author = {Agogino, Adrian K. and Lee, Ritchie and Giannakopoulou, Dimitra},
title = {Machine Learning Explainability and Transferability for Path Navigation},
booktitle = aiaa_is,
year = {2021},
organization = {AIAA}
}

@Article{Corso2021a,
author = {Anthony Corso and Robert J. Moss and Mark Koren and Ritchie Lee and Mykel J. Kochenderfer},
title = {A Survey of Algorithms for Black-box Safety Validation},
journal = jair,
year = {2021},
volume = {72},
pages = {377--428}
}

@InProceedings{Durling2021a,
title = {Certification Considerations for Adaptive Stress Testing of Airborne Software},
author = {Durling, Michael and Herencia-Zapana, Heber and Meng, Baoluo and Meiners, Mike and Hochwarth, Joachim and Visser, Nicholas and Lee, Ritchie and Moss, Robert and Valapil, Vidhya Tekken},
booktitle = dasc,
year = {2021},
organization = {AIAA/IEEE}
}

@inproceedings{Lipkis2022,
title = {Adaptive Stress Testing of Collision Avoidance Systems for Small UASs with Deep Reinforcement Learning},
author = {Lipkis, Rory and Lee, Ritchie and Silbermann, Joshua and Young, Tyler},
booktitle = aiaa_is,
organization = {AIAA},
year = {2022}
}

ExprOptimization.jl

ExprOptimization.jl is a software package written in Julia for the optimization of expression trees. The package can be used to learn expressions or executable source code from input/output examples, also known as “program induction”. The package implements a number of tree optimization algorithms including Genetic Programming, Monte Carlo, and Grammatical Evolution. The software has been used to learn interpretable rules in multivariate heterogeneous time series datasets.

Related Publications:

R. Lee and L. Shen, System identification of a Cessna 182 model UAV, 2007.
[Bibtex]

@Unpublished{Lee2007,
author = {Ritchie Lee and Lu Shen},
title = {System Identification of a {C}essna 182 Model {UAV}},
organization = {{S}tanford {U}niversity CS229},
year = {2007}
}

R. Lee and C. Ippolito, “A perception and mapping approach for plume detection in payload directed flight,” in Aiaa [email protected] conference, 2009.
[Bibtex]

@InProceedings{Lee2009,
author = {Lee, Ritchie and Ippolito, Corey},
title = {A Perception and Mapping Approach for Plume Detection in Payload Directed Flight},
booktitle = aiaa_info,
organization = {AIAA},
year = {2009},
month = {April}
}

R. Lee, C. Ippolito, Y. Yeh, J. Spritzer, and G. Phelps, “Payload-directed control of geophysical magnetic surveys,” in Aiaa [email protected] conference, 2010.
[Bibtex]

@InProceedings{Lee2010a,
Title = {Payload-Directed Control of Geophysical Magnetic Surveys},
Author = {Lee, Ritchie and Ippolito, Corey and Yeh, Yoo-Hsiu and Spritzer, John and Phelps, Geoffrey},
Booktitle = aiaa_info,
organization = {AIAA},
Year = {2010},
Month = {April}
}

R. Lee and D. H. Wolpert, “Game theoretic modeling of human behavior in mid-air encounters,” in NIPS workshop on decision making with multiple imperfect decision makers, 2010.
[Bibtex]

@InProceedings{Lee2010b,
Title = {Game Theoretic Modeling of Human Behavior in Mid-Air Encounters},
Author = {Lee, Ritchie and Wolpert, David H},
Booktitle = {{NIPS} Workshop on Decision Making with Multiple Imperfect Decision Makers},
Year = {2010}
}

@InProceedings{Dolan2011,
author = {Dolan, Jacques and Lee, Ritchie and Yeh, Yoo-Hsiu and Yeh, Chiping and Nguyen, Daniel Y and Ben-Menahem, Shahar and Ishihara, Abraham K},
title = {Neural Network Estimation of Photovoltaic {I}--{V} Curves Under Partially Shaded Conditions},
booktitle = ijcnn,
year = {2011},
pages = {1358--1365},
organization = {IEEE}
}

@InProceedings{Lee2011,
author = {Lee, Ritchie and Backhaus, Scott and Bono, James and Wolpert, David H. and Bent, Russell and Tracey, Brendan},
title = {Modeling Humans as Reinforcement Learners: How to Predict Human Behavior in Multi-Stage Games},
booktitle = {{NIPS} Workshop on Decision Making with Multiple Imperfect Decision Makers},
year = {2011}
}

R. Lee and D. H. Wolpert, “Game theoretic modeling of pilot behavior during mid-air encounters,” in Decision making with imperfect decision makers, Springer, 2012, vol. 28, pp. 75-111.
[Bibtex]

@InCollection{Lee2012,
author = {Lee, Ritchie and Wolpert, David H.},
title = {Game Theoretic Modeling of Pilot Behavior During Mid-Air Encounters},
booktitle = {Decision Making with Imperfect Decision Makers},
publisher = {Springer},
year = {2012},
volume = {28},
series = {Intelligent Systems Reference Library},
chapter = {4},
pages = {75-111}
}

E. Denney, C. Ippolito, R. Lee, and G. Pai, “An integrated safety and systems engineering methodology for small unmanned aircraft systems,” in Aiaa [email protected] conference, 2012.
[Bibtex]

@InProceedings{Denney2012,
Title = {An Integrated Safety and Systems Engineering Methodology for Small Unmanned Aircraft Systems},
Author = {Ewen Denney and Corey Ippolito and Ritchie Lee and Ganesh Pai},
Booktitle = aiaa_info,
Organization = {AIAA},
Year = {2012}
}

@InProceedings{Schlicht2012,
Title = {Predicting the Behavior of Interacting Humans by Fusing Data from Multiple Sources},
Author = {Erik J. Schlicht and Ritchie Lee and David H. Wolpert and Mykel J. Kochenderfer and Brendan Tracey},
Booktitle = uai,
Organization = {AUAI},
Year = {2012}
}

@InProceedings{Yan2012,
Title = {Towards a {B}ayesian Network Game Framework for Evaluating {DDoS} Attacks and Defense},
Author = {Yan, Guanhua and Lee, Ritchie and Kent, Alex and Wolpert, David H.},
Booktitle = acmccs,
Year = {2012},
Organization = {ACM},
Pages = {553--566}
}

@InProceedings{Yildiz2012,
Title = {Using Game Theoretic Models to Predict Pilot Behavior in {N}ext{G}en Merging and Landing Scenario},
Author = {Yildiz, Yildiray and Lee, Ritchie and Brat, Guillaume},
Booktitle = aiaa_mst,
Organization = {AIAA},
Year = {2012}
}

@InCollection{Lee2013,
author = {Lee, Ritchie and Wolpert, David H. and Bono, James and Backhaus, Scott and Bent, Russell and Tracey, Brendan},
title = {Counter-Factual Reinforcement Learning: How to Model Decision-Makers That Anticipate the Future},
booktitle = {Decision Making and Imperfection},
publisher = {Springer},
year = {2013},
volume = {474},
series = {Studies in Computational Intelligence},
chapter = {4},
pages = {101-128}
}

@Article{Backhaus2013,
author = {Backhaus, Scott and Bent, Russell and Bono, James and Lee, Ritchie and Tracey, Brendan and Wolpert, David H. and Xie, Dongping and Yildiz, Yildiray},
title = {Cyber-Physical Security: A Game Theory Model of Humans Interacting Over Control Systems},
journal = ieeesg,
year = {2013},
volume = {4},
number = {4},
pages = {2320--2327},
publisher = {IEEE}
}

R. Lee and M. Rajabi, Assessing NuPIC and CLA in a machine learning context using NASA aviation datasets, 2014.
[Bibtex]

@Unpublished{Lee2014,
author = {Ritchie Lee and Mariam Rajabi},
title = {Assessing {NuPIC} and {CLA} in a Machine Learning Context Using {NASA} Aviation Datasets},
organization = {Carnegie Mellon University EE18799-SV},
year = {2014}
}

@TechReport{Phelps2014,
author = {Phelps, Geoffrey A and Ippolito, Corey and Lee, Ritchie and Spritzer, John and Yeh, Yoo-Hsiu},
title = {Investigations into Near-Real-Time Surveying for Geophysical Data Collection Using an Autonomous Ground Vehicle},
institution = usgs,
year = {2014}
}

@InProceedings{Lee2015,
author = {Ritchie Lee and Mykel J. Kochenderfer and Ole J. Mengshoel and Guillaume P. Brat and Michael P. Owen},
title = {Adaptive Stress Testing of Airborne Collision Avoidance Systems},
booktitle = dasc,
organization = {AIAA/IEEE},
note = {Best Paper of Session},
year = {2015}
}

@InProceedings{Agogino2015,
Title = {Scalable Hierarchical Multifidelity Simulation and Multiagent Optimization of Air Traffic},
Author = {Adrian Agogino and Atil Iscen and Ritchie Lee and Devin Bowers and Kagan Tumer and Guillaume Brat},
Booktitle = {{AAMAS} Workshop on Massive Multiagent Systems},
Year = {2015}
}

R. Lee, A. Agogino, and D. Giannakopoulou, “Algorithms and tools for model-based V&V of autonomy software,” NASA 2017.
[Bibtex]

@TechReport{Lee2017,
Title = {Algorithms and Tools for Model-Based {V\&V} of Autonomy Software},
Author = {Lee, Ritchie and Agogino, Adrian and Giannakopoulou, Dimitra},
Institution = {NASA},
Year = {2017}
}

@InProceedings{Lee2018a,
author = {Lee, Ritchie and Mengshoel, Ole J. and Saksena, Anshu and Gardner, Ryan and Genin, Daniel and Brush, Jeffrey and Kochenderfer, Mykel J.},
title = {Differential Adaptive Stress Testing of Airborne Collision Avoidance Systems},
booktitle = aiaa_mst,
year = {2018},
note = {Best Paper Award},
Organization = {AIAA}
}

R. Lee, M. J. Kochenderfer, O. J. Mengshoel, and J. Silbermann, “Interpretable categorization of heterogeneous time series data,” in International conference on data mining (sdm), 2018.
[Bibtex]

@InProceedings{Lee2018b,
author = {Lee, Ritchie and Kochenderfer, Mykel J. and Mengshoel, Ole J. and Silbermann, Joshua},
title = {Interpretable Categorization of Heterogeneous Time Series Data},
booktitle = sdm,
year = {2018},
Organization = {SIAM}
}

M. Koren, S. Alsaif, R. Lee, and M. J. Kochenderfer, “Adaptive stress testing for autonomous vehicles,” in Ieee intelligent vehicles symposium (iv), 2018.
[Bibtex]

@InProceedings{Koren2018,
author = {Mark Koren and Saud Alsaif and Ritchie Lee and Mykel J. Kochenderfer},
title = {Adaptive Stress Testing for Autonomous Vehicles},
booktitle = iv,
Organization = {IEEE},
year = {2018}
}

@inproceedings{Lee2019a,
author = {Lee, Ritchie and Mengshoel, Ole J. and Agogino, Adrian K. and Giannakopoulou, Dimitra and Kochenderfer, Mykel J.},
title = {Adaptive Stress Testing of Trajectory Planning Systems},
booktitle = aiaa_is,
year = {2019},
organization = {AIAA}
}

A. K. Agogino, R. Lee, and D. Giannakopoulou, “Challenges of explaining real-time planning,” in ICAPS workshop on explainable planning (XAIP), 2019.
[Bibtex]

@InProceedings{Agogino2019a,
Author = {Agogino, Adrian K. and Lee, Ritchie and Giannakopoulou, Dimitra},
title = {Challenges of Explaining Real-Time Planning},
booktitle = {{ICAPS} Workshop on Explainable Planning ({XAIP})},
year = {2019}
}

R. Lee, “Adastress: adaptive stress testing and interpretable categorization for safety-critical systems,” PhD Thesis, 2019.
[Bibtex]

@phdthesis{leethesis,
author = {Ritchie Lee},
title = {AdaStress: Adaptive Stress Testing and Interpretable Categorization for Safety-Critical Systems},
school = {Carnegie Mellon University},
year = {2019},
month = {May}
}

R. Lee, O. J. Mengshoel, and M. J. Kochenderfer, “Adaptive stress testing of safety-critical systems,” in Safe, autonomous, and intelligent vehicles, Springer, 2019, p. 77–95.
[Bibtex]

@InCollection{Lee2019b,
author = {Lee, Ritchie and Mengshoel, Ole J. and Kochenderfer, Mykel J.},
title = {Adaptive Stress Testing of Safety-Critical Systems},
booktitle = {Safe, Autonomous, and Intelligent Vehicles},
series = {Unmanned Systems Technology},
publisher = {Springer},
year = {2019},
pages = {77--95}
}

@inproceedings{Lee2020a,
author = {Lee, Ritchie and Clarke, Justin, and Agogino, Adrian K. and Giannakopoulou, Dimitra},
title = {Improving Trust in Deep Neural Networks using Nearest Neighbors},
booktitle = aiaa_is,
year = {2020},
organization = {AIAA}
}

@InProceedings{Moss2020a,
author = {Robert J. Moss and Ritchie Lee and Mykel J. Kochenderfer},
title = {Adaptive Stress Testing of Trajectory Predictions in Flight Management Systems},
booktitle = dasc,
organization = {AIAA/IEEE},
note = {Best Paper of Session},
year = {2020}
}

A. Corso, R. Lee, and M. J. Kochenderfer, “Scalable autonomous vehicle safety validation through dynamic programming and scene decomposition,” in Ieee international conference on intelligent transportation systems (itsc), 2020.
[Bibtex]
```
@InProceedings{Corso2020a,
author = {Anthony Corso and Ritchie Lee and Mykel J. Kochenderfer},
booktitle = itsc,
title = {Scalable Autonomous Vehicle Safety Validation Through Dynamic Programming and Scene Decomposition},
year = {2020}
}
```
K. D. Julian, R. Lee, and M. J. Kochenderfer, “Validation of image-based neural network controllers through adaptive stress testing,” in Ieee international conference on intelligent transportation systems (itsc), 2020.
[Bibtex]
```
@InProceedings{Julian2020a,
author = {Kyle D. Julian and Ritchie Lee and Mykel J. Kochenderfer},
booktitle = itsc,
title = {Validation of Image-Based Neural Network Controllers Through Adaptive Stress Testing},
year = {2020}
}
```

@article{Lee2020b,
author = {Lee, Ritchie and Mengshoel, Ole J. and Saksena, Anshu and Gardner, Ryan and Genin, Daniel and Silbermann, Joshua and Owen, Michael and Kochenderfer, Mykel J.},
title = {Adaptive Stress Testing: Finding Likely Failure Events with Reinforcement Learning},
journal = jair,
year = {2020},
volume = {69},
pages = {1165--1201}
}

Proc. 12th international NASA formal methods symposium, R. Lee, S. Jha, A. Mavridou, and D. Giannakoupoulou, Eds., Springer, 2020, vol. 12229.
[Bibtex]

@book{Lee2020c,
title = {Proc. 12th International {NASA} Formal Methods Symposium},
editor = {Ritchie Lee and Susmit Jha and Anastasia Mavridou and Dimitra Giannakoupoulou},
series = {Lecture Notes in Computer Science},
volume = {12229},
publisher = {Springer},
year = {2020},
}

A. K. Agogino, R. Lee, and D. Giannakopoulou, “Machine learning explainability and transferability for path navigation,” in Aiaa scitech, intelligent systems conference (is), 2021.
[Bibtex]

@inproceedings{Agogino2021a,
author = {Agogino, Adrian K. and Lee, Ritchie and Giannakopoulou, Dimitra},
title = {Machine Learning Explainability and Transferability for Path Navigation},
booktitle = aiaa_is,
year = {2021},
organization = {AIAA}
}

@Article{Corso2021a,
author = {Anthony Corso and Robert J. Moss and Mark Koren and Ritchie Lee and Mykel J. Kochenderfer},
title = {A Survey of Algorithms for Black-box Safety Validation},
journal = jair,
year = {2021},
volume = {72},
pages = {377--428}
}

@InProceedings{Durling2021a,
title = {Certification Considerations for Adaptive Stress Testing of Airborne Software},
author = {Durling, Michael and Herencia-Zapana, Heber and Meng, Baoluo and Meiners, Mike and Hochwarth, Joachim and Visser, Nicholas and Lee, Ritchie and Moss, Robert and Valapil, Vidhya Tekken},
booktitle = dasc,
year = {2021},
organization = {AIAA/IEEE}
}

@inproceedings{Lipkis2022,
title = {Adaptive Stress Testing of Collision Avoidance Systems for Small UASs with Deep Reinforcement Learning},
author = {Lipkis, Rory and Lee, Ritchie and Silbermann, Joshua and Young, Tyler},
booktitle = aiaa_is,
organization = {AIAA},
year = {2022}
}

Multi-Fidelity Simulator (MFSim)

MFSim is a fast-time national airspace simulator written in Java. The simulator uses a hierarchical backend that allows aircraft in different regions of the airspace to be simulated at different levels of fidelity. The simulator has been used as a platform for research in multi-agent reinforcement learning of air traffic automation.

Related Publications:

R. Lee and L. Shen, System identification of a Cessna 182 model UAV, 2007.
[Bibtex]

@Unpublished{Lee2007,
author = {Ritchie Lee and Lu Shen},
title = {System Identification of a {C}essna 182 Model {UAV}},
organization = {{S}tanford {U}niversity CS229},
year = {2007}
}

R. Lee and C. Ippolito, “A perception and mapping approach for plume detection in payload directed flight,” in Aiaa [email protected] conference, 2009.
[Bibtex]

@InProceedings{Lee2009,
author = {Lee, Ritchie and Ippolito, Corey},
title = {A Perception and Mapping Approach for Plume Detection in Payload Directed Flight},
booktitle = aiaa_info,
organization = {AIAA},
year = {2009},
month = {April}
}

R. Lee, C. Ippolito, Y. Yeh, J. Spritzer, and G. Phelps, “Payload-directed control of geophysical magnetic surveys,” in Aiaa [email protected] conference, 2010.
[Bibtex]

@InProceedings{Lee2010a,
Title = {Payload-Directed Control of Geophysical Magnetic Surveys},
Author = {Lee, Ritchie and Ippolito, Corey and Yeh, Yoo-Hsiu and Spritzer, John and Phelps, Geoffrey},
Booktitle = aiaa_info,
organization = {AIAA},
Year = {2010},
Month = {April}
}

R. Lee and D. H. Wolpert, “Game theoretic modeling of human behavior in mid-air encounters,” in NIPS workshop on decision making with multiple imperfect decision makers, 2010.
[Bibtex]

@InProceedings{Lee2010b,
Title = {Game Theoretic Modeling of Human Behavior in Mid-Air Encounters},
Author = {Lee, Ritchie and Wolpert, David H},
Booktitle = {{NIPS} Workshop on Decision Making with Multiple Imperfect Decision Makers},
Year = {2010}
}

@InProceedings{Dolan2011,
author = {Dolan, Jacques and Lee, Ritchie and Yeh, Yoo-Hsiu and Yeh, Chiping and Nguyen, Daniel Y and Ben-Menahem, Shahar and Ishihara, Abraham K},
title = {Neural Network Estimation of Photovoltaic {I}--{V} Curves Under Partially Shaded Conditions},
booktitle = ijcnn,
year = {2011},
pages = {1358--1365},
organization = {IEEE}
}

@InProceedings{Lee2011,
author = {Lee, Ritchie and Backhaus, Scott and Bono, James and Wolpert, David H. and Bent, Russell and Tracey, Brendan},
title = {Modeling Humans as Reinforcement Learners: How to Predict Human Behavior in Multi-Stage Games},
booktitle = {{NIPS} Workshop on Decision Making with Multiple Imperfect Decision Makers},
year = {2011}
}

R. Lee and D. H. Wolpert, “Game theoretic modeling of pilot behavior during mid-air encounters,” in Decision making with imperfect decision makers, Springer, 2012, vol. 28, pp. 75-111.
[Bibtex]

@InCollection{Lee2012,
author = {Lee, Ritchie and Wolpert, David H.},
title = {Game Theoretic Modeling of Pilot Behavior During Mid-Air Encounters},
booktitle = {Decision Making with Imperfect Decision Makers},
publisher = {Springer},
year = {2012},
volume = {28},
series = {Intelligent Systems Reference Library},
chapter = {4},
pages = {75-111}
}

E. Denney, C. Ippolito, R. Lee, and G. Pai, “An integrated safety and systems engineering methodology for small unmanned aircraft systems,” in Aiaa [email protected] conference, 2012.
[Bibtex]

@InProceedings{Denney2012,
Title = {An Integrated Safety and Systems Engineering Methodology for Small Unmanned Aircraft Systems},
Author = {Ewen Denney and Corey Ippolito and Ritchie Lee and Ganesh Pai},
Booktitle = aiaa_info,
Organization = {AIAA},
Year = {2012}
}

@InProceedings{Schlicht2012,
Title = {Predicting the Behavior of Interacting Humans by Fusing Data from Multiple Sources},
Author = {Erik J. Schlicht and Ritchie Lee and David H. Wolpert and Mykel J. Kochenderfer and Brendan Tracey},
Booktitle = uai,
Organization = {AUAI},
Year = {2012}
}

@InProceedings{Yan2012,
Title = {Towards a {B}ayesian Network Game Framework for Evaluating {DDoS} Attacks and Defense},
Author = {Yan, Guanhua and Lee, Ritchie and Kent, Alex and Wolpert, David H.},
Booktitle = acmccs,
Year = {2012},
Organization = {ACM},
Pages = {553--566}
}

@InProceedings{Yildiz2012,
Title = {Using Game Theoretic Models to Predict Pilot Behavior in {N}ext{G}en Merging and Landing Scenario},
Author = {Yildiz, Yildiray and Lee, Ritchie and Brat, Guillaume},
Booktitle = aiaa_mst,
Organization = {AIAA},
Year = {2012}
}

@InCollection{Lee2013,
author = {Lee, Ritchie and Wolpert, David H. and Bono, James and Backhaus, Scott and Bent, Russell and Tracey, Brendan},
title = {Counter-Factual Reinforcement Learning: How to Model Decision-Makers That Anticipate the Future},
booktitle = {Decision Making and Imperfection},
publisher = {Springer},
year = {2013},
volume = {474},
series = {Studies in Computational Intelligence},
chapter = {4},
pages = {101-128}
}

@Article{Backhaus2013,
author = {Backhaus, Scott and Bent, Russell and Bono, James and Lee, Ritchie and Tracey, Brendan and Wolpert, David H. and Xie, Dongping and Yildiz, Yildiray},
title = {Cyber-Physical Security: A Game Theory Model of Humans Interacting Over Control Systems},
journal = ieeesg,
year = {2013},
volume = {4},
number = {4},
pages = {2320--2327},
publisher = {IEEE}
}

R. Lee and M. Rajabi, Assessing NuPIC and CLA in a machine learning context using NASA aviation datasets, 2014.
[Bibtex]

@Unpublished{Lee2014,
author = {Ritchie Lee and Mariam Rajabi},
title = {Assessing {NuPIC} and {CLA} in a Machine Learning Context Using {NASA} Aviation Datasets},
organization = {Carnegie Mellon University EE18799-SV},
year = {2014}
}

@TechReport{Phelps2014,
author = {Phelps, Geoffrey A and Ippolito, Corey and Lee, Ritchie and Spritzer, John and Yeh, Yoo-Hsiu},
title = {Investigations into Near-Real-Time Surveying for Geophysical Data Collection Using an Autonomous Ground Vehicle},
institution = usgs,
year = {2014}
}

@InProceedings{Lee2015,
author = {Ritchie Lee and Mykel J. Kochenderfer and Ole J. Mengshoel and Guillaume P. Brat and Michael P. Owen},
title = {Adaptive Stress Testing of Airborne Collision Avoidance Systems},
booktitle = dasc,
organization = {AIAA/IEEE},
note = {Best Paper of Session},
year = {2015}
}

@InProceedings{Agogino2015,
Title = {Scalable Hierarchical Multifidelity Simulation and Multiagent Optimization of Air Traffic},
Author = {Adrian Agogino and Atil Iscen and Ritchie Lee and Devin Bowers and Kagan Tumer and Guillaume Brat},
Booktitle = {{AAMAS} Workshop on Massive Multiagent Systems},
Year = {2015}
}

R. Lee, A. Agogino, and D. Giannakopoulou, “Algorithms and tools for model-based V&V of autonomy software,” NASA 2017.
[Bibtex]

@TechReport{Lee2017,
Title = {Algorithms and Tools for Model-Based {V\&V} of Autonomy Software},
Author = {Lee, Ritchie and Agogino, Adrian and Giannakopoulou, Dimitra},
Institution = {NASA},
Year = {2017}
}

@InProceedings{Lee2018a,
author = {Lee, Ritchie and Mengshoel, Ole J. and Saksena, Anshu and Gardner, Ryan and Genin, Daniel and Brush, Jeffrey and Kochenderfer, Mykel J.},
title = {Differential Adaptive Stress Testing of Airborne Collision Avoidance Systems},
booktitle = aiaa_mst,
year = {2018},
note = {Best Paper Award},
Organization = {AIAA}
}

R. Lee, M. J. Kochenderfer, O. J. Mengshoel, and J. Silbermann, “Interpretable categorization of heterogeneous time series data,” in International conference on data mining (sdm), 2018.
[Bibtex]

@InProceedings{Lee2018b,
author = {Lee, Ritchie and Kochenderfer, Mykel J. and Mengshoel, Ole J. and Silbermann, Joshua},
title = {Interpretable Categorization of Heterogeneous Time Series Data},
booktitle = sdm,
year = {2018},
Organization = {SIAM}
}

M. Koren, S. Alsaif, R. Lee, and M. J. Kochenderfer, “Adaptive stress testing for autonomous vehicles,” in Ieee intelligent vehicles symposium (iv), 2018.
[Bibtex]

@InProceedings{Koren2018,
author = {Mark Koren and Saud Alsaif and Ritchie Lee and Mykel J. Kochenderfer},
title = {Adaptive Stress Testing for Autonomous Vehicles},
booktitle = iv,
Organization = {IEEE},
year = {2018}
}

@inproceedings{Lee2019a,
author = {Lee, Ritchie and Mengshoel, Ole J. and Agogino, Adrian K. and Giannakopoulou, Dimitra and Kochenderfer, Mykel J.},
title = {Adaptive Stress Testing of Trajectory Planning Systems},
booktitle = aiaa_is,
year = {2019},
organization = {AIAA}
}

A. K. Agogino, R. Lee, and D. Giannakopoulou, “Challenges of explaining real-time planning,” in ICAPS workshop on explainable planning (XAIP), 2019.
[Bibtex]

@InProceedings{Agogino2019a,
Author = {Agogino, Adrian K. and Lee, Ritchie and Giannakopoulou, Dimitra},
title = {Challenges of Explaining Real-Time Planning},
booktitle = {{ICAPS} Workshop on Explainable Planning ({XAIP})},
year = {2019}
}

R. Lee, “Adastress: adaptive stress testing and interpretable categorization for safety-critical systems,” PhD Thesis, 2019.
[Bibtex]

@phdthesis{leethesis,
author = {Ritchie Lee},
title = {AdaStress: Adaptive Stress Testing and Interpretable Categorization for Safety-Critical Systems},
school = {Carnegie Mellon University},
year = {2019},
month = {May}
}

R. Lee, O. J. Mengshoel, and M. J. Kochenderfer, “Adaptive stress testing of safety-critical systems,” in Safe, autonomous, and intelligent vehicles, Springer, 2019, p. 77–95.
[Bibtex]

@InCollection{Lee2019b,
author = {Lee, Ritchie and Mengshoel, Ole J. and Kochenderfer, Mykel J.},
title = {Adaptive Stress Testing of Safety-Critical Systems},
booktitle = {Safe, Autonomous, and Intelligent Vehicles},
series = {Unmanned Systems Technology},
publisher = {Springer},
year = {2019},
pages = {77--95}
}

@inproceedings{Lee2020a,
author = {Lee, Ritchie and Clarke, Justin, and Agogino, Adrian K. and Giannakopoulou, Dimitra},
title = {Improving Trust in Deep Neural Networks using Nearest Neighbors},
booktitle = aiaa_is,
year = {2020},
organization = {AIAA}
}

@InProceedings{Moss2020a,
author = {Robert J. Moss and Ritchie Lee and Mykel J. Kochenderfer},
title = {Adaptive Stress Testing of Trajectory Predictions in Flight Management Systems},
booktitle = dasc,
organization = {AIAA/IEEE},
note = {Best Paper of Session},
year = {2020}
}

A. Corso, R. Lee, and M. J. Kochenderfer, “Scalable autonomous vehicle safety validation through dynamic programming and scene decomposition,” in Ieee international conference on intelligent transportation systems (itsc), 2020.
[Bibtex]
```
@InProceedings{Corso2020a,
author = {Anthony Corso and Ritchie Lee and Mykel J. Kochenderfer},
booktitle = itsc,
title = {Scalable Autonomous Vehicle Safety Validation Through Dynamic Programming and Scene Decomposition},
year = {2020}
}
```
K. D. Julian, R. Lee, and M. J. Kochenderfer, “Validation of image-based neural network controllers through adaptive stress testing,” in Ieee international conference on intelligent transportation systems (itsc), 2020.
[Bibtex]
```
@InProceedings{Julian2020a,
author = {Kyle D. Julian and Ritchie Lee and Mykel J. Kochenderfer},
booktitle = itsc,
title = {Validation of Image-Based Neural Network Controllers Through Adaptive Stress Testing},
year = {2020}
}
```

@article{Lee2020b,
author = {Lee, Ritchie and Mengshoel, Ole J. and Saksena, Anshu and Gardner, Ryan and Genin, Daniel and Silbermann, Joshua and Owen, Michael and Kochenderfer, Mykel J.},
title = {Adaptive Stress Testing: Finding Likely Failure Events with Reinforcement Learning},
journal = jair,
year = {2020},
volume = {69},
pages = {1165--1201}
}

Proc. 12th international NASA formal methods symposium, R. Lee, S. Jha, A. Mavridou, and D. Giannakoupoulou, Eds., Springer, 2020, vol. 12229.
[Bibtex]

@book{Lee2020c,
title = {Proc. 12th International {NASA} Formal Methods Symposium},
editor = {Ritchie Lee and Susmit Jha and Anastasia Mavridou and Dimitra Giannakoupoulou},
series = {Lecture Notes in Computer Science},
volume = {12229},
publisher = {Springer},
year = {2020},
}

A. K. Agogino, R. Lee, and D. Giannakopoulou, “Machine learning explainability and transferability for path navigation,” in Aiaa scitech, intelligent systems conference (is), 2021.
[Bibtex]

@inproceedings{Agogino2021a,
author = {Agogino, Adrian K. and Lee, Ritchie and Giannakopoulou, Dimitra},
title = {Machine Learning Explainability and Transferability for Path Navigation},
booktitle = aiaa_is,
year = {2021},
organization = {AIAA}
}

@Article{Corso2021a,
author = {Anthony Corso and Robert J. Moss and Mark Koren and Ritchie Lee and Mykel J. Kochenderfer},
title = {A Survey of Algorithms for Black-box Safety Validation},
journal = jair,
year = {2021},
volume = {72},
pages = {377--428}
}

@InProceedings{Durling2021a,
title = {Certification Considerations for Adaptive Stress Testing of Airborne Software},
author = {Durling, Michael and Herencia-Zapana, Heber and Meng, Baoluo and Meiners, Mike and Hochwarth, Joachim and Visser, Nicholas and Lee, Ritchie and Moss, Robert and Valapil, Vidhya Tekken},
booktitle = dasc,
year = {2021},
organization = {AIAA/IEEE}
}

@inproceedings{Lipkis2022,
title = {Adaptive Stress Testing of Collision Avoidance Systems for Small UASs with Deep Reinforcement Learning},
author = {Lipkis, Rory and Lee, Ritchie and Silbermann, Joshua and Young, Tyler},
booktitle = aiaa_is,
organization = {AIAA},
year = {2022}
}

Network-Form Games Library (libnfg)

Libnfg is a software library written in C++ for implementing Network-Form Games (NFGs). NFGs model systems with multiple human actors whose strategic decisions interact in the context of the system. The algorithms use game theory, probabilistic inference, and reinforcement learning to predict the dynamics of the overall system.

Related Publications:

R. Lee and L. Shen, System identification of a Cessna 182 model UAV, 2007.
[Bibtex]

@Unpublished{Lee2007,
author = {Ritchie Lee and Lu Shen},
title = {System Identification of a {C}essna 182 Model {UAV}},
organization = {{S}tanford {U}niversity CS229},
year = {2007}
}

R. Lee and C. Ippolito, “A perception and mapping approach for plume detection in payload directed flight,” in Aiaa [email protected] conference, 2009.
[Bibtex]

@InProceedings{Lee2009,
author = {Lee, Ritchie and Ippolito, Corey},
title = {A Perception and Mapping Approach for Plume Detection in Payload Directed Flight},
booktitle = aiaa_info,
organization = {AIAA},
year = {2009},
month = {April}
}

R. Lee, C. Ippolito, Y. Yeh, J. Spritzer, and G. Phelps, “Payload-directed control of geophysical magnetic surveys,” in Aiaa [email protected] conference, 2010.
[Bibtex]

@InProceedings{Lee2010a,
Title = {Payload-Directed Control of Geophysical Magnetic Surveys},
Author = {Lee, Ritchie and Ippolito, Corey and Yeh, Yoo-Hsiu and Spritzer, John and Phelps, Geoffrey},
Booktitle = aiaa_info,
organization = {AIAA},
Year = {2010},
Month = {April}
}

R. Lee and D. H. Wolpert, “Game theoretic modeling of human behavior in mid-air encounters,” in NIPS workshop on decision making with multiple imperfect decision makers, 2010.
[Bibtex]

@InProceedings{Lee2010b,
Title = {Game Theoretic Modeling of Human Behavior in Mid-Air Encounters},
Author = {Lee, Ritchie and Wolpert, David H},
Booktitle = {{NIPS} Workshop on Decision Making with Multiple Imperfect Decision Makers},
Year = {2010}
}

@InProceedings{Dolan2011,
author = {Dolan, Jacques and Lee, Ritchie and Yeh, Yoo-Hsiu and Yeh, Chiping and Nguyen, Daniel Y and Ben-Menahem, Shahar and Ishihara, Abraham K},
title = {Neural Network Estimation of Photovoltaic {I}--{V} Curves Under Partially Shaded Conditions},
booktitle = ijcnn,
year = {2011},
pages = {1358--1365},
organization = {IEEE}
}

@InProceedings{Lee2011,
author = {Lee, Ritchie and Backhaus, Scott and Bono, James and Wolpert, David H. and Bent, Russell and Tracey, Brendan},
title = {Modeling Humans as Reinforcement Learners: How to Predict Human Behavior in Multi-Stage Games},
booktitle = {{NIPS} Workshop on Decision Making with Multiple Imperfect Decision Makers},
year = {2011}
}

R. Lee and D. H. Wolpert, “Game theoretic modeling of pilot behavior during mid-air encounters,” in Decision making with imperfect decision makers, Springer, 2012, vol. 28, pp. 75-111.
[Bibtex]

@InCollection{Lee2012,
author = {Lee, Ritchie and Wolpert, David H.},
title = {Game Theoretic Modeling of Pilot Behavior During Mid-Air Encounters},
booktitle = {Decision Making with Imperfect Decision Makers},
publisher = {Springer},
year = {2012},
volume = {28},
series = {Intelligent Systems Reference Library},
chapter = {4},
pages = {75-111}
}

E. Denney, C. Ippolito, R. Lee, and G. Pai, “An integrated safety and systems engineering methodology for small unmanned aircraft systems,” in Aiaa [email protected] conference, 2012.
[Bibtex]

@InProceedings{Denney2012,
Title = {An Integrated Safety and Systems Engineering Methodology for Small Unmanned Aircraft Systems},
Author = {Ewen Denney and Corey Ippolito and Ritchie Lee and Ganesh Pai},
Booktitle = aiaa_info,
Organization = {AIAA},
Year = {2012}
}

@InProceedings{Schlicht2012,
Title = {Predicting the Behavior of Interacting Humans by Fusing Data from Multiple Sources},
Author = {Erik J. Schlicht and Ritchie Lee and David H. Wolpert and Mykel J. Kochenderfer and Brendan Tracey},
Booktitle = uai,
Organization = {AUAI},
Year = {2012}
}

@InProceedings{Yan2012,
Title = {Towards a {B}ayesian Network Game Framework for Evaluating {DDoS} Attacks and Defense},
Author = {Yan, Guanhua and Lee, Ritchie and Kent, Alex and Wolpert, David H.},
Booktitle = acmccs,
Year = {2012},
Organization = {ACM},
Pages = {553--566}
}

@InProceedings{Yildiz2012,
Title = {Using Game Theoretic Models to Predict Pilot Behavior in {N}ext{G}en Merging and Landing Scenario},
Author = {Yildiz, Yildiray and Lee, Ritchie and Brat, Guillaume},
Booktitle = aiaa_mst,
Organization = {AIAA},
Year = {2012}
}

@InCollection{Lee2013,
author = {Lee, Ritchie and Wolpert, David H. and Bono, James and Backhaus, Scott and Bent, Russell and Tracey, Brendan},
title = {Counter-Factual Reinforcement Learning: How to Model Decision-Makers That Anticipate the Future},
booktitle = {Decision Making and Imperfection},
publisher = {Springer},
year = {2013},
volume = {474},
series = {Studies in Computational Intelligence},
chapter = {4},
pages = {101-128}
}

@Article{Backhaus2013,
author = {Backhaus, Scott and Bent, Russell and Bono, James and Lee, Ritchie and Tracey, Brendan and Wolpert, David H. and Xie, Dongping and Yildiz, Yildiray},
title = {Cyber-Physical Security: A Game Theory Model of Humans Interacting Over Control Systems},
journal = ieeesg,
year = {2013},
volume = {4},
number = {4},
pages = {2320--2327},
publisher = {IEEE}
}

R. Lee and M. Rajabi, Assessing NuPIC and CLA in a machine learning context using NASA aviation datasets, 2014.
[Bibtex]

@Unpublished{Lee2014,
author = {Ritchie Lee and Mariam Rajabi},
title = {Assessing {NuPIC} and {CLA} in a Machine Learning Context Using {NASA} Aviation Datasets},
organization = {Carnegie Mellon University EE18799-SV},
year = {2014}
}

@TechReport{Phelps2014,
author = {Phelps, Geoffrey A and Ippolito, Corey and Lee, Ritchie and Spritzer, John and Yeh, Yoo-Hsiu},
title = {Investigations into Near-Real-Time Surveying for Geophysical Data Collection Using an Autonomous Ground Vehicle},
institution = usgs,
year = {2014}
}

@InProceedings{Lee2015,
author = {Ritchie Lee and Mykel J. Kochenderfer and Ole J. Mengshoel and Guillaume P. Brat and Michael P. Owen},
title = {Adaptive Stress Testing of Airborne Collision Avoidance Systems},
booktitle = dasc,
organization = {AIAA/IEEE},
note = {Best Paper of Session},
year = {2015}
}

@InProceedings{Agogino2015,
Title = {Scalable Hierarchical Multifidelity Simulation and Multiagent Optimization of Air Traffic},
Author = {Adrian Agogino and Atil Iscen and Ritchie Lee and Devin Bowers and Kagan Tumer and Guillaume Brat},
Booktitle = {{AAMAS} Workshop on Massive Multiagent Systems},
Year = {2015}
}

R. Lee, A. Agogino, and D. Giannakopoulou, “Algorithms and tools for model-based V&V of autonomy software,” NASA 2017.
[Bibtex]

@TechReport{Lee2017,
Title = {Algorithms and Tools for Model-Based {V\&V} of Autonomy Software},
Author = {Lee, Ritchie and Agogino, Adrian and Giannakopoulou, Dimitra},
Institution = {NASA},
Year = {2017}
}

@InProceedings{Lee2018a,
author = {Lee, Ritchie and Mengshoel, Ole J. and Saksena, Anshu and Gardner, Ryan and Genin, Daniel and Brush, Jeffrey and Kochenderfer, Mykel J.},
title = {Differential Adaptive Stress Testing of Airborne Collision Avoidance Systems},
booktitle = aiaa_mst,
year = {2018},
note = {Best Paper Award},
Organization = {AIAA}
}

R. Lee, M. J. Kochenderfer, O. J. Mengshoel, and J. Silbermann, “Interpretable categorization of heterogeneous time series data,” in International conference on data mining (sdm), 2018.
[Bibtex]

@InProceedings{Lee2018b,
author = {Lee, Ritchie and Kochenderfer, Mykel J. and Mengshoel, Ole J. and Silbermann, Joshua},
title = {Interpretable Categorization of Heterogeneous Time Series Data},
booktitle = sdm,
year = {2018},
Organization = {SIAM}
}

M. Koren, S. Alsaif, R. Lee, and M. J. Kochenderfer, “Adaptive stress testing for autonomous vehicles,” in Ieee intelligent vehicles symposium (iv), 2018.
[Bibtex]

@InProceedings{Koren2018,
author = {Mark Koren and Saud Alsaif and Ritchie Lee and Mykel J. Kochenderfer},
title = {Adaptive Stress Testing for Autonomous Vehicles},
booktitle = iv,
Organization = {IEEE},
year = {2018}
}

@inproceedings{Lee2019a,
author = {Lee, Ritchie and Mengshoel, Ole J. and Agogino, Adrian K. and Giannakopoulou, Dimitra and Kochenderfer, Mykel J.},
title = {Adaptive Stress Testing of Trajectory Planning Systems},
booktitle = aiaa_is,
year = {2019},
organization = {AIAA}
}

A. K. Agogino, R. Lee, and D. Giannakopoulou, “Challenges of explaining real-time planning,” in ICAPS workshop on explainable planning (XAIP), 2019.
[Bibtex]

@InProceedings{Agogino2019a,
Author = {Agogino, Adrian K. and Lee, Ritchie and Giannakopoulou, Dimitra},
title = {Challenges of Explaining Real-Time Planning},
booktitle = {{ICAPS} Workshop on Explainable Planning ({XAIP})},
year = {2019}
}

R. Lee, “Adastress: adaptive stress testing and interpretable categorization for safety-critical systems,” PhD Thesis, 2019.
[Bibtex]

@phdthesis{leethesis,
author = {Ritchie Lee},
title = {AdaStress: Adaptive Stress Testing and Interpretable Categorization for Safety-Critical Systems},
school = {Carnegie Mellon University},
year = {2019},
month = {May}
}

R. Lee, O. J. Mengshoel, and M. J. Kochenderfer, “Adaptive stress testing of safety-critical systems,” in Safe, autonomous, and intelligent vehicles, Springer, 2019, p. 77–95.
[Bibtex]

@InCollection{Lee2019b,
author = {Lee, Ritchie and Mengshoel, Ole J. and Kochenderfer, Mykel J.},
title = {Adaptive Stress Testing of Safety-Critical Systems},
booktitle = {Safe, Autonomous, and Intelligent Vehicles},
series = {Unmanned Systems Technology},
publisher = {Springer},
year = {2019},
pages = {77--95}
}

@inproceedings{Lee2020a,
author = {Lee, Ritchie and Clarke, Justin, and Agogino, Adrian K. and Giannakopoulou, Dimitra},
title = {Improving Trust in Deep Neural Networks using Nearest Neighbors},
booktitle = aiaa_is,
year = {2020},
organization = {AIAA}
}

@InProceedings{Moss2020a,
author = {Robert J. Moss and Ritchie Lee and Mykel J. Kochenderfer},
title = {Adaptive Stress Testing of Trajectory Predictions in Flight Management Systems},
booktitle = dasc,
organization = {AIAA/IEEE},
note = {Best Paper of Session},
year = {2020}
}

A. Corso, R. Lee, and M. J. Kochenderfer, “Scalable autonomous vehicle safety validation through dynamic programming and scene decomposition,” in Ieee international conference on intelligent transportation systems (itsc), 2020.
[Bibtex]
```
@InProceedings{Corso2020a,
author = {Anthony Corso and Ritchie Lee and Mykel J. Kochenderfer},
booktitle = itsc,
title = {Scalable Autonomous Vehicle Safety Validation Through Dynamic Programming and Scene Decomposition},
year = {2020}
}
```
K. D. Julian, R. Lee, and M. J. Kochenderfer, “Validation of image-based neural network controllers through adaptive stress testing,” in Ieee international conference on intelligent transportation systems (itsc), 2020.
[Bibtex]
```
@InProceedings{Julian2020a,
author = {Kyle D. Julian and Ritchie Lee and Mykel J. Kochenderfer},
booktitle = itsc,
title = {Validation of Image-Based Neural Network Controllers Through Adaptive Stress Testing},
year = {2020}
}
```

@article{Lee2020b,
author = {Lee, Ritchie and Mengshoel, Ole J. and Saksena, Anshu and Gardner, Ryan and Genin, Daniel and Silbermann, Joshua and Owen, Michael and Kochenderfer, Mykel J.},
title = {Adaptive Stress Testing: Finding Likely Failure Events with Reinforcement Learning},
journal = jair,
year = {2020},
volume = {69},
pages = {1165--1201}
}

Proc. 12th international NASA formal methods symposium, R. Lee, S. Jha, A. Mavridou, and D. Giannakoupoulou, Eds., Springer, 2020, vol. 12229.
[Bibtex]

@book{Lee2020c,
title = {Proc. 12th International {NASA} Formal Methods Symposium},
editor = {Ritchie Lee and Susmit Jha and Anastasia Mavridou and Dimitra Giannakoupoulou},
series = {Lecture Notes in Computer Science},
volume = {12229},
publisher = {Springer},
year = {2020},
}

A. K. Agogino, R. Lee, and D. Giannakopoulou, “Machine learning explainability and transferability for path navigation,” in Aiaa scitech, intelligent systems conference (is), 2021.
[Bibtex]

@inproceedings{Agogino2021a,
author = {Agogino, Adrian K. and Lee, Ritchie and Giannakopoulou, Dimitra},
title = {Machine Learning Explainability and Transferability for Path Navigation},
booktitle = aiaa_is,
year = {2021},
organization = {AIAA}
}

@Article{Corso2021a,
author = {Anthony Corso and Robert J. Moss and Mark Koren and Ritchie Lee and Mykel J. Kochenderfer},
title = {A Survey of Algorithms for Black-box Safety Validation},
journal = jair,
year = {2021},
volume = {72},
pages = {377--428}
}

@InProceedings{Durling2021a,
title = {Certification Considerations for Adaptive Stress Testing of Airborne Software},
author = {Durling, Michael and Herencia-Zapana, Heber and Meng, Baoluo and Meiners, Mike and Hochwarth, Joachim and Visser, Nicholas and Lee, Ritchie and Moss, Robert and Valapil, Vidhya Tekken},
booktitle = dasc,
year = {2021},
organization = {AIAA/IEEE}
}

@inproceedings{Lipkis2022,
title = {Adaptive Stress Testing of Collision Avoidance Systems for Small UASs with Deep Reinforcement Learning},
author = {Lipkis, Rory and Lee, Ritchie and Silbermann, Joshua and Young, Tyler},
booktitle = aiaa_is,
organization = {AIAA},
year = {2022}
}