Coverage for dragonfly_sim/core/windtunnel_model.py: 85%

560 statements  

« prev     ^ index     » next       coverage.py v7.16.2, created at 2026-10-21 00:13 +0000

1import numpy as np 

2from mpi4py import MPI 

3from petsc4py import PETSc 

4import dolfinx 

5 

6import datetime 

7import resource 

8import os 

9import gc 

10 

11import csdl_alpha as csdl 

12 

13from time import perf_counter 

14 

15from dragonfly_sim.core.mesh_manager import Mesh 

16from dragonfly_sim.core.Euler_model import CompressibleEulerModel 

17from dragonfly_sim.core.meshwarping import IDWarp_jax 

18from dragonfly_sim.utils.data_handler import DataStore 

19 

20from dragonfly_sim.utils.petsc_utils import set_petsc_vec_array 

21from dragonfly_sim.utils.derived_fields import destroy_chained_operator 

22from dragonfly_sim.utils.solver_utils import ksp_converged_reason_name 

23from dragonfly_sim.core.postprocessor import DG_postprocessor 

24 

25 

26# TODO: Look into using entropy variables instead of conservative variables; 

27# these would potentially allow us to do simulations from low Mach all 

28# the way to supersonic 

29class DG_windtunnel_model(csdl.experimental.CustomImplicitOperation): 

30 # The solve/adjoint callbacks contain MPI collectives (PETSc, mpi4py) and deform 

31 # the shared mesh, so they must never overlap -- also on a single process. 

32 ordered_callbacks = True 

33 

34 def __init__(self, mesh, boundary_dict, shape_parameterization=None, moments=None, 

35 mesh_inner_bdry_function=None, wall_tags=None, symmetry_tags=None, 

36 poly_order=1, gamma=1.4, 

37 filename_suffix="test", 

38 asm_overlap=None, ilu_levels=None, 

39 model_class=CompressibleEulerModel, model_kwargs=None, 

40 reduced_order_model=None, compute_all_adjoint_errors=False): 

41 """ 

42 boundary_dict: the free-stream state {'rho', 'p', 'M', 'alpha'} (density, 

43 pressure, Mach number, angle of attack in rad). The whole outer boundary 

44 is a characteristic far field built from it, so the angle of attack can 

45 change by any amount during an optimization; there is no inflow/outflow 

46 division to choose or to keep valid. 

47 

48 The shape of the object being optimized (the wall) is defined in one of 

49 three ways, in this order of precedence: 

50 1. `mesh_inner_bdry_function`, a function of the boundary facet midpoints 

51 (shape (dim, n)) returning a boolean wall mask. It can encode 

52 exclusions the FFD block cannot. If the mesh also carries boundary 

53 facet tags (utils/mesh_io_utils.py:load_dolfinx_mesh stores them on 

54 the mesh), a warning lists them: they could be used instead. 

55 2. `wall_tags`: the mesh's boundary facet tags (values or names, one or 

56 several) that are the wall, and `symmetry_tags` those that are 

57 symmetry planes (slip walls). Every other boundary is far field. 

58 In 3D without tags, the symmetry plane is y = 0. 

59 3. The shape parameterization's FFD block corners 

60 (shape_parameterization.inner_boundary_function()), which bound the 

61 box the inner boundary facets are contained in. 

62 At least one is required. 

63 

64 shape_parameterization: the wall-shape parameterization, e.g. a 

65 shape_design.FFDShapeParameterization -- the FFD block and the layers 

66 that define the shape design variables and their bounds. The model does 

67 not need to know what those variables are: set_up_sim builds the IDWarp 

68 mesh warper (self.mesh_warper) and hands the wall to 

69 shape_parameterization.setup(); deform_mesh() turns its wall 

70 displacement into mesh node motions; evaluate() takes those and the 

71 flat shape parameter vector: shape optimization. Without it (None) no 

72 warper is built, the mesh stays fixed, and evaluate() takes alpha only: 

73 a forward flow analysis, with derivatives with respect to alpha. 

74 solve_forward() runs one flow solve without a CSDL graph at all. 

75 

76 moments: the moments of the wall force to compute, print and differentiate, 

77 a sequence of (name, point, direction[, ref_length]) tuples (see 

78 DG_postprocessor); default none. E.g. the pitching moment about the quarter 

79 chord of a wing, ("pitch", (0.25, 0., 0.), "pitch"), and its root 

80 bending moment, ("root_bending", (0.25, 0., 0.), (1., 0., 0.)). 

81 

82 model_class / model_kwargs select the flow model: CompressibleEulerModel 

83 (the default) or e.g. core.RANS_model.CompressibleRANSModel with 

84 model_kwargs={'Re': 6e6}. The body's wall is the model's 

85 define_wall_bc (slip for Euler, adiabatic no-slip for RANS). 

86 

87 reduced_order_model: a core.reduced_order_model.ReducedOrderModel to 

88 try before every flow solve (see _solve), or None for full-order 

89 solves only. Where its solution is accepted, the derivatives use the 

90 adjoint of its projected equations (see apply_inverse_jacobian). 

91 

92 compute_all_adjoint_errors: diagnostic, default False. When a derivative 

93 uses the ROM adjoint, also solve the full-order adjoint at the same state 

94 and right-hand side and record the ROM adjoint's error against it in 

95 data_store.adjoint_errors. The adjoint handed to the optimizer is the 

96 ROM's either way; the extra full-order solve costs about as much as the 

97 adjoint the ROM saves, and is not part of data_store.adjoint_solves. 

98 """ 

99 super().__init__() 

100 

101 self.asm_overlap = asm_overlap 

102 self.ilu_levels = ilu_levels 

103 

104 # --- Linear-solve convergence criteria ---------------------------- 

105 # We use one set of convergence criteria for all linear solves. 

106 # This does not cover the Newton solver for the Euler model itself, 

107 # but all other things (adjoint, Helmholtz, elasticity-based  

108 # mesh warping, etc.) when used. 

109 self.linear_solver_rtol = 1e-20 

110 self.linear_solver_atol = 1e-11 

111 self.linear_solver_max_it = 1000 

112 

113 # If linear solves converge above this tolerance, we raise an error 

114 self.linear_residual_accept_limit = 1e-10 

115 

116 # --- Nonlinear (Euler) convergence criteria ----------------------- 

117 self.euler_residual_conv_limit = 1e-9 

118 # Total Newton-step budget for the FOM solve 

119 self.max_newton_iterations = 100 

120 # KDTree coincidence radius for the dof <-> mesh-node permutation. 

121 # These two orderings are 1:1 by construction and differ only in 

122 # index order, so this tolerance accounts for round-off errors 

123 self.mesh_node_match_tol = 1e-10 

124 

125 if shape_parameterization is not None and not hasattr(shape_parameterization, 'setup'): 

126 raise TypeError("shape_parameterization must be a shape parameterization object " 

127 "(e.g. shape_design.FFDShapeParameterization), got {!r}. The FFD " 

128 "block arguments (ffd_shape, ffd_degree, ffd_block_corner_list, " 

129 "cp_coord_opt_idxs) moved there.".format(shape_parameterization)) 

130 if mesh_inner_bdry_function is None and wall_tags is None and shape_parameterization is not None: 

131 mesh_inner_bdry_function = shape_parameterization.inner_boundary_function() 

132 if mesh_inner_bdry_function is None and wall_tags is None: 

133 raise ValueError("Provide either mesh_inner_bdry_function, wall_tags (the mesh's own " 

134 "boundary tags) or a shape parameterization with FFD block corners " 

135 "to locate the inner boundary facets.") 

136 

137 # Initial instantiation of mesh and simulation model 

138 self.mesh = Mesh(mesh, mesh_inner_bdry_function, 

139 node_match_tol=self.mesh_node_match_tol, 

140 wall_tags=wall_tags, symmetry_tags=symmetry_tags) 

141 if model_class is CompressibleEulerModel: 

142 _euler_kwargs = dict(poly_order=poly_order, gamma=gamma) 

143 self.sim_model = CompressibleEulerModel(self.mesh, **_euler_kwargs) 

144 else: 

145 self.sim_model = model_class(self.mesh, gamma=gamma, **(model_kwargs or {})) 

146 # TODO: Perhaps split up CompressibleEulerModel into an Euler model class and a solver class that facilitates solver interactions;  

147 # the solver has become so complicated recently that spinning it off into its own class would likely help with code readability 

148 

149 if asm_overlap is not None: 

150 self.sim_model.asm_overlap = asm_overlap 

151 if ilu_levels is not None: 

152 self.sim_model.ilu_levels = ilu_levels 

153 

154 # Set by solve_residual_equations, once, as the LAST write to this 

155 # name for the evaluation: whether the solve that produced the 

156 # accepted u_vec actually converged.  

157 # Mainly used for testing purposes. 

158 self.sim_model.last_solve_converged = False 

159 

160 # Mesh warping only with a shape parameterization (see __init__) 

161 self.shape_parameterization = shape_parameterization 

162 self.shape_parameterized = shape_parameterization is not None 

163 

164 # The mesh warper, built in set_up_sim() 

165 self.mesh_warper = None 

166 

167 # Define data storage object for external post-processing 

168 self.data_store = DataStore() 

169 

170 self.previous_solution = None 

171 

172 # POD reduced-order model (None: full-order solves only) 

173 self.reduced_order_model = reduced_order_model 

174 self.compute_all_adjoint_errors = compute_all_adjoint_errors 

175 # (eval_idx, number of adjoint solves so far at it): the output row of 

176 # the derivative evaluation that CSDL is on 

177 self._adjoint_row = (None, 0) 

178 # (parameter vector, converged) of the last evaluation if the FOM 

179 # solved it, else None: a re-evaluation of that design skips the ROM 

180 # (see _solve) 

181 self._fom_design = None 

182 

183 # Naming suffix for exported files 

184 self.filename_suffix = filename_suffix 

185 

186 # Free the flow solver's matrices after every forward solve and the 

187 # condensed dR/dU after every condensed adjoint 

188 # (sim_model.release_solver), so the two never coexist; they are 

189 # rebuilt when next needed. For memory-bound runs (3D RANS). 

190 self.release_solver_memory = False 

191 

192 # Adjoint reuse. The transposed dR/dU and the preconditioned KSP of the 

193 # state last linearized (condensed Jacobian), kept for the adjoint 

194 # solves of every output of that design: {'key': state key, 'ksp': KSP}; 

195 # freed when the next flow solve starts or a different state is 

196 # linearized. 

197 self._adjoint_cache = None 

198 

199 # Options of the adjoint of a model with a condensed Jacobian (default off; 

200 # set_adjoint_options sets them together with those of the reconstruction): 

201 # adjoint_matrix_free the adjoint operator A_UU^T x + B^T (A_UG^T x), without 

202 # forming A or the product A_UG B 

203 # cache_jacvec_operators the dR/dx operator and dR/dalpha of the 

204 # compute_jacvec_product calls of all outputs of one 

205 # design are built once (freed with the adjoint cache) 

206 self.adjoint_matrix_free = False 

207 self.cache_jacvec_operators = False 

208 # PETSc options (names without the "adj_" prefix) overriding the adjoint KSP's settings of the 

209 # model, e.g. {"ksp_type": "hpddm", "ksp_hpddm_type": "gcrodr", "ksp_hpddm_recycle": 20}; the KSP 

210 # is kept for all outputs of a design, so a recycling method carries its subspace between them 

211 self.adjoint_ksp_options = None 

212 self._dRdx_cache = None 

213 self._dRdalpha_cache = None 

214 

215 # Start each condensed adjoint solve from a first-order solution: the 

216 # solution of the same transposed system with the compact (nearest 

217 # neighbour, gradients frozen) part A_UU of the Jacobian alone -- the 

218 # first-order Jacobian the preconditioner is built from -- solved to a 

219 # relative tolerance first_order_adjoint_rtol. It reuses the 

220 # preconditioner's factors, so costs only Krylov iterations. Applies to 

221 # models with a condensed Jacobian (MUSCL / RANS); the guess is always 

222 # used, and its effect is logged (self._last_first_order_guess). 

223 self.adjoint_first_order_guess = False 

224 self.first_order_adjoint_rtol = 1e-2 

225 self._last_first_order_guess = None 

226 

227 # Write mesh_deformation_<suffix>.bp (the node motions of every design 

228 # evaluation). The solution files are written either way. 

229 self.write_mesh_deformation = True 

230 

231 # Write the output frame (solution and mesh deformation) only for the 

232 # first evaluation of each design: an optimizer that evaluates the 

233 # same design again (CSDL re-runs the model for the derivatives) then 

234 # does not duplicate the frame. Frames keep the eval_idx of that first 

235 # evaluation, so their labels have gaps. 

236 self.write_once_per_design = False 

237 self._last_written_design = None 

238 

239 

240 # Discretization and physics-related parameters 

241 self.poly_order:int = poly_order 

242 self.gamma:float = gamma 

243 self.boundary_condition_dict = boundary_dict 

244 self.moments = moments 

245 

246 # Dolfinx mesh tags for the various boundaries 

247 self.WALL_TAG = 2 # walls (including wing/airfoil boundaries) 

248 self.SYMMETRY_TAG = 5 # symmetry planes (3D: y = 0, or the mesh's symmetry_tags) 

249 self.FARFIELD_TAG = 6 # characteristic far field: the whole outer boundary 

250 

251 # global evaluation index of solve_residual_equations 

252 self.eval_idx = 0 

253 

254 def set_up_sim(self): 

255 # pass the various tolerances and limits to the Euler sim model 

256 self.sim_model.residual_conv_limit = self.euler_residual_conv_limit 

257 self.sim_model.max_newton_iterations = self.max_newton_iterations 

258 

259 # use the free-stream dictionary to define the far-field conditions 

260 self.sim_model.define_farfield_conditions(self.boundary_condition_dict) 

261 

262 # compute boundary facet boolean arrays 

263 inner_facets = self.mesh.inner_bdry_facet_mask 

264 

265 # Find the symmetry face facets (only relevant in 3D at the moment) 

266 # NOTE: The symmetry plane is currently hardcoded at y=0 

267 has_symmetry = self.mesh.symmetry_facet_mask is not None or self.sim_model.dimensions == 3 

268 if self.mesh.symmetry_facet_mask is not None: 

269 symmetry_facets = self.mesh.symmetry_facet_mask # from the mesh's boundary tags 

270 elif self.sim_model.dimensions == 3: 

271 symmetry_facets = self.mesh.bdry_midpoints[:, 1] <= 1e-6 

272 else: 

273 symmetry_facets = np.array([False]*inner_facets.shape[0]) 

274 

275 # Everything that is neither wall nor symmetry plane is far field. The 

276 # characteristic far-field state decides facet by facet whether the flow 

277 # enters or leaves, so no inflow/outflow division (and nothing to 

278 # re-tag when the angle of attack changes) is needed. 

279 farfield_facets = ~np.asarray(inner_facets, dtype=bool) & ~np.asarray(symmetry_facets, dtype=bool) 

280 

281 # Print the local number of facets of each type for each MPI rank 

282 PETSc.Sys.syncPrint("Rank {}, number of wing boundary facets: {}".format(self.mesh.mesh.comm.Get_rank(), np.count_nonzero(inner_facets))) 

283 PETSc.Sys.syncPrint("Rank {}, number of symmetry facets: {}".format(self.mesh.mesh.comm.Get_rank(), np.count_nonzero(symmetry_facets))) 

284 PETSc.Sys.syncPrint("Rank {}, number of far-field facets: {}".format(self.mesh.mesh.comm.Get_rank(), np.count_nonzero(farfield_facets))) 

285 PETSc.Sys.syncFlush() 

286 

287 # Pair the boundary facet sets with the various mesh tags 

288 bdry_meshtags_and_ds_list = [(farfield_facets, self.FARFIELD_TAG), (inner_facets, self.WALL_TAG)] 

289 if has_symmetry: 

290 bdry_meshtags_and_ds_list += [(symmetry_facets, self.SYMMETRY_TAG)] 

291 

292 # Build the boundary facet tags and define the corresponding quadrature measure 

293 self.mesh.tag_boundary_facets(bdry_meshtags_and_ds_list) 

294 self.mesh.form_manager.build_boundary_measure( 

295 measure_metadata={'quadrature_degree': 3*self.poly_order + 1}) 

296 

297 self.mesh.configure_output(self.filename_suffix, 

298 deformation=self.shape_parameterized and self.write_mesh_deformation) 

299 

300 if self.shape_parameterized: 

301 # The JAX-based IDWarp implementation as mesh warper 

302 self.mesh_warper = IDWarp_jax(self.mesh, wall_meshtag_val=self.WALL_TAG, 

303 anchor_meshtag_vals=(self.FARFIELD_TAG,)) 

304 

305 PETSc.Sys.Print("Initialized mesh warping object") 

306 

307 # The shape parameterization gets the wall nodes (replicated, in the 

308 # numbering of the mesh's wall node set) and facets; its 

309 # wall_displacement() is the (M_global, dim) array mesh_warper.evaluate 

310 # takes. 

311 wall_nodes = self.mesh.boundary_node_set((self.WALL_TAG,)) 

312 self.shape_parameterization.setup(self.mesh.mesh.comm, wall_nodes.coords, 

313 wall_nodes.global_idx, 

314 self.mesh.wall_facets(self.WALL_TAG)) 

315 else: 

316 PETSc.Sys.Print("No shape parameterization given: fixed mesh, forward flow analysis") 

317 

318 # define the integration measure for the mesh cells and non-boundary facets 

319 self.mesh.form_manager.build_cell_and_interior_facet_measures( 

320 measure_metadata={'quadrature_degree': 3*self.poly_order + 1}) 

321 

322 # Define the required Dolfinx objects for the simulation, including the initial conditions 

323 self.sim_model.define_elements_functionspaces() 

324 self.sim_model.define_trial_testfunctions() 

325 self.sim_model.compute_initial_conditions_from_farfield() 

326 self.sim_model.interpolate_solution_vector() 

327 

328 # We output the facet mesh tags to an xdmf file, so we can inspect the BCs in paraview 

329 self.mesh.mesh.topology.create_connectivity(self.sim_model.dimensions-1, self.sim_model.dimensions) 

330 with dolfinx.io.XDMFFile(self.mesh.mesh.comm, "output_meshtags.xdmf", "w") as xdmf: 

331 # Write the Mesh object to the file 

332 xdmf.write_mesh(self.mesh.mesh) 

333 # Write the MeshTags object to the file 

334 xdmf.write_meshtags(self.mesh.meshtags, self.mesh.mesh.geometry) 

335 

336 # Define the boundary condition terms of the Euler weak form 

337 self.sim_model.define_farfield_bc(self.FARFIELD_TAG) 

338 self.sim_model.define_wall_bc(self.WALL_TAG) 

339 if has_symmetry: 

340 self.sim_model.define_slipwall_bc(self.SYMMETRY_TAG) 

341 

342 # Construct the Euler weak form for forward evaluation 

343 self.sim_model.compute_weakform() 

344 

345 # Define the file names and objects for solution writes to external files for Paraview 

346 self.sim_model.define_sol_export_files(self.sim_model.functionspaces["V"], 

347 file_name_addendum=self.filename_suffix, 

348 mesh_deformation=self.shape_parameterized) 

349 

350 # We instantiate a separate DG_postprocessor object here so we can print  

351 # forces and moments and coefficients after each forward solve 

352 self.postprocessor = DG_postprocessor( 

353 self.mesh, self.sim_model, self.WALL_TAG, 

354 moments=self.moments, 

355 p_inf_dim=101325.) 

356 

357 if self.reduced_order_model is not None: 

358 self.reduced_order_model.set_up(self.sim_model) 

359 

360 def _log_memory(self, stage): 

361 # We write a log file for each MPI rank that documents the peak local memory usage 

362 # at various points during an optimization run. 

363 mem = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss 

364 rank = MPI.COMM_WORLD.Get_rank() 

365 fname = 'memory_logs/r{}_{}.log'.format(rank, self.filename_suffix) 

366 os.makedirs('memory_logs', exist_ok=True) 

367 with open(fname, 'a') as f: 

368 f.write('{} {} {} {} {}\n'.format( 

369 self.eval_idx, datetime.datetime.now(), os.getpid(), mem, stage)) 

370 

371 @property 

372 def ffd(self): 

373 """The shape parameterization's FFD block (None without one, or before 

374 set_up_sim).""" 

375 if self.shape_parameterization is None: 

376 return None 

377 return getattr(self.shape_parameterization, 'ffd', None) 

378 

379 def wall_geometry(self): 

380 """Geometric constraint quantities of the FFD-deformed wall (area/volume, 

381 thickness, planform); the shape parameterization's 

382 shape_parameterization.WallGeometry. 

383 

384 Built on first use. Collective: call it on every rank. 

385 """ 

386 if self.ffd is None: 

387 raise RuntimeError("wall_geometry needs a shape parameterization (an FFD block), " 

388 "set up by set_up_sim") 

389 return self.shape_parameterization.geometry 

390 

391 def deform_mesh(self): 

392 """Mesh node motions (CSDL) for the shape parameterization's current 

393 wall displacement: the first input of evaluate(). Call after the 

394 shape design variables are declared.""" 

395 if not self.shape_parameterized: 

396 raise RuntimeError("deform_mesh needs a shape parameterization") 

397 shape = self.shape_parameterization 

398 return self.mesh_warper.evaluate(shape.wall_displacement(), shape.parameter_vector()) 

399 

400 def evaluate(self, mesh_node_motions:csdl.Variable = None, shape_parameters:csdl.Variable = None, 

401 alpha:csdl.Variable = None): 

402 """ 

403 The flow state u_vec as a CSDL output. 

404 

405 Shape optimization (a shape parameterization was given): 

406 evaluate(mesh_node_motions, shape_parameters, alpha), with 

407 mesh_node_motions from deform_mesh() and shape_parameters the flat 

408 shape parameter vector. Forward analysis on the fixed mesh: 

409 evaluate(alpha=alpha). alpha is always required. 

410 """ 

411 if alpha is None: 

412 raise ValueError("evaluate needs alpha") 

413 if (mesh_node_motions is None) != (shape_parameters is None): 

414 raise ValueError("pass mesh_node_motions and shape_parameters together, or neither") 

415 if mesh_node_motions is not None and not self.shape_parameterized: 

416 raise ValueError("mesh_node_motions given, but there is no shape parameterization / " 

417 "mesh warper (pass shape_parameterization to DG_windtunnel_model)") 

418 

419 # set inputs using self.declare_input 

420 if mesh_node_motions is not None: 

421 self.declare_input('mesh_node_motions', mesh_node_motions) 

422 self.declare_input('shape_parameters', shape_parameters) 

423 self.declare_input('alpha', alpha) 

424 

425 # The output on this rank is only the local portion of the solution vector 

426 u_vec = self.create_output('u_vec', (self.sim_model.u_vec.x.petsc_vec.local_size,)) 

427 

428 if mesh_node_motions is not None: 

429 # We have declared shape_parameters as an input here, just for logging and computation purposes, 

430 # not because we actually need it; mesh_node_motions carries the effect of the shape_parameters. 

431 self.declare_derivative_parameters(of='u_vec', wrt='shape_parameters', dependent=False) 

432 

433 return u_vec 

434 

435 def solve_forward(self, alpha=None, snapshot=True): 

436 """ 

437 One flow solve on the current (undeformed) mesh, without CSDL: set 

438 alpha (radians; default: the boundary dictionary's), solve, print the 

439 coefficients. Warm-starts from the previous solve. Call set_up_sim() 

440 first. Returns (u_vec array, converged, forces), forces being 

441 DG_postprocessor.forces_and_coefficients (L, D, c_l, c_d, the moments, and 

442 the friction drag for a viscous model). 

443 

444 The ROM (if any) stores the solve as a snapshot at the parameter vector 

445 (alpha). With shape parameters that vector is longer, so a solve for a 

446 baseline that is not a design of the optimization passes snapshot=False. 

447 """ 

448 if alpha is not None: 

449 self.sim_model.set_angle_of_attack(alpha) 

450 self.postprocessor.set_angle_of_attack(alpha) 

451 self.sim_model.compute_weakform() 

452 rom, self.reduced_order_model = self.reduced_order_model, (self.reduced_order_model if snapshot else None) 

453 try: 

454 solution, converged = self._solve( 

455 self._parameter_vector(None, self.sim_model.farfield_conditions['alpha'])) 

456 finally: 

457 self.reduced_order_model = rom 

458 self.sim_model.last_solve_converged = converged 

459 if np.isfinite(np.linalg.norm(solution)): 

460 self.previous_solution = solution 

461 return solution, converged, self.postprocessor.forces_and_coefficients(solution) 

462 

463 def solve_residual_equations(self, input_vals, output_vals): 

464 # Define inputs 

465 mesh_node_motions = input_vals['mesh_node_motions'] if 'mesh_node_motions' in input_vals else None 

466 shape_parameters = input_vals['shape_parameters'] if 'shape_parameters' in input_vals else None 

467 alpha = input_vals['alpha'][0] 

468 

469 # Update the angle of attack to its current value 

470 self.sim_model.set_angle_of_attack(alpha) 

471 

472 PETSc.Sys.Print("angle of attack: {} deg".format(np.degrees(alpha))) 

473 PETSc.Sys.Print("shape parameters: {}".format(shape_parameters)) 

474 PETSc.Sys.Print("solve_residual_equations, stepping into mesh.apply_node_motions") 

475 if mesh_node_motions is not None: 

476 PETSc.Sys.Print("mesh_node_motions 2-norm: {}".format(np.linalg.norm(mesh_node_motions))) 

477 

478 # Apply the mesh node motions to the mesh 

479 self.mesh.apply_node_motions(mesh_node_motions, design=shape_parameters) 

480 

481 # If the mesh is not valid (e.g. has inverted cells), set the output to zero and don't run any simulations 

482 if not self.mesh.is_valid: 

483 PETSc.Sys.Print("Mesh is not valid, skipping simulations.") 

484 output_vals['u_vec'] = np.zeros((self.sim_model.u_vec.x.petsc_vec.local_size,)) 

485 self._fom_design = None 

486 if self.reduced_order_model is not None: 

487 self.reduced_order_model.discard_adjoint() 

488 return 

489 

490 # re-define the current weak form if necessary 

491 self.sim_model.compute_weakform() 

492 

493 # --- flow solve (ROM, else FOM) ----------------------------------- 

494 solution_output, solution_converged = self._solve(self._parameter_vector(shape_parameters, alpha)) 

495 

496 self._log_memory("after flow solve, before setting u_vec output") 

497 

498 self.sim_model.last_solve_converged = solution_converged 

499 

500 PETSc.Sys.Print("setting u_vec output...") 

501 output_vals['u_vec'] = solution_output 

502 

503 # Warm start: keep the accepted solution unless it has blown up 

504 if np.isfinite(np.linalg.norm(solution_output)): 

505 PETSc.Sys.Print("Updating warm start solution") 

506 self.previous_solution = solution_output 

507 else: 

508 PETSc.Sys.Print("Not updating warm start solution, using previous value") 

509 

510 design = np.concatenate([[alpha], np.ravel(shape_parameters) 

511 if shape_parameters is not None else []]) 

512 if (self.write_once_per_design and self._last_written_design is not None 

513 and np.array_equal(design, self._last_written_design)): 

514 PETSc.Sys.Print("Same design as the last written frame: not writing eval_idx {}".format( 

515 self.eval_idx)) 

516 else: 

517 PETSc.Sys.Print("Writing aligned output frame at eval_idx {} (FOM solution)".format( 

518 self.eval_idx)) 

519 # we write the deformation and the solution outputs to the same time step indices 

520 if mesh_node_motions is not None: 

521 self.mesh.write_deformation_output(mesh_node_motions, self.eval_idx) 

522 self.sim_model.write_solution_output(self.eval_idx, 

523 solution_array=solution_output) 

524 self._last_written_design = design 

525 

526 # Explicitly collect garbage and cleanup PETSc objects 

527 gc.collect() 

528 PETSc.garbage_cleanup(self.mesh.mesh.comm) 

529 

530 self._log_memory("after Euler PDE solve") 

531 

532 self.eval_idx += 1 

533 

534 @staticmethod 

535 def _parameter_vector(shape_parameters, alpha): 

536 """The ROM snapshot parameter vector of an evaluation: the flat shape 

537 parameters (if any), then alpha.""" 

538 parts = [] if shape_parameters is None else [np.asarray(shape_parameters, dtype=float).ravel()] 

539 return np.concatenate(parts + [np.array([float(alpha)])]) 

540 

541 def _solve(self, parameter_vector): 

542 """ 

543 The accepted flow solution of one evaluation: (owned solution entries, 

544 converged). 

545 

546 Without a reduced-order model this is the full-order (FOM) solve from 

547 the warm start. With one, the ROM is tried first (once it holds enough 

548 snapshots), from the warm start projected onto the basis, and its 

549 solution is accepted when eta = ||R||/||u|| < eta_threshold; the ROM 

550 then holds the adjoint of that solution (rom.adjoint), which 

551 apply_inverse_jacobian uses. Otherwise the FOM solves from the warm 

552 start exactly as without a ROM, and its solution, if converged and 

553 finite, becomes a snapshot at `parameter_vector`. 

554 

555 A re-evaluation of the design the FOM solved last skips the ROM: the 

556 FOM restarts from its own solution (no Newton steps if that 

557 converged), so the design keeps the FOM solution and the full-order 

558 adjoint. CSDL re-runs the model before every derivative evaluation, 

559 and by then the ROM would reproduce that solution, now a snapshot, and 

560 accept it. 

561 """ 

562 # the forward solve needs the memory of the last design's adjoint operators 

563 self._release_adjoint_cache() 

564 rom = self.reduced_order_model 

565 if rom is None: 

566 return self._run_fom_solve() 

567 rom.discard_adjoint() 

568 

569 repeat = self._fom_design is not None and np.array_equal(parameter_vector, self._fom_design[0]) 

570 rom_result = rom_forces = None 

571 if repeat: 

572 PETSc.Sys.Print("Re-evaluating the design the FOM solved last: skipping the ROM") 

573 else: 

574 self.sim_model.interpolate_solution_vector(self.previous_solution) 

575 rom_result = rom.solve(self.sim_model, parameter_vector) 

576 if rom_result is not None: 

577 rom_forces = self._log_rom_solve(rom_result) 

578 accept_rom = rom_result is not None and rom_result.accepted 

579 if accept_rom and not rom.run_fom_every_evaluation: 

580 PETSc.Sys.Print("Accepting ROM solution") 

581 self._fom_design = None 

582 return rom_result.solution, True 

583 

584 solution_fom, fom_converged = self._run_fom_solve() 

585 if rom_forces is not None and fom_converged: 

586 self._log_rom_error(rom_forces) 

587 self._log_rom_state_error(rom_result.solution, solution_fom) 

588 if accept_rom: 

589 PETSc.Sys.Print("Accepting ROM solution (FOM run for comparison)") 

590 self._fom_design = None 

591 return rom_result.solution, True 

592 

593 PETSc.Sys.Print("Accepting FOM solution") 

594 finite = self.mesh.mesh.comm.allreduce(bool(np.all(np.isfinite(solution_fom))), op=MPI.LAND) 

595 converged = fom_converged and finite 

596 if converged and repeat and self._fom_design[1]: 

597 PETSc.Sys.Print("The snapshot of this design is already stored") 

598 elif converged: 

599 self.sim_model.set_u_vec(solution_fom) 

600 rom.add_snapshot(self.sim_model, parameter_vector) 

601 sigma = rom.snapshot_singular_values() 

602 if self.mesh.mesh.comm.Get_rank() == 0 and sigma is not None: 

603 self.data_store.global_snapshot_sing_vals_per_iteration += [(self.eval_idx, sigma)] 

604 else: 

605 PETSc.Sys.Print("Not adding a ROM snapshot: the FOM solve did not converge") 

606 self._fom_design = (np.array(parameter_vector, dtype=float), converged) 

607 return solution_fom, fom_converged 

608 

609 def _log_rom_solve(self, result): 

610 # coefficients of the ROM solution, logged like the FOM's 

611 outputs, (cd, cl, *cms) = self._evaluate_outputs(result.solution) 

612 cms = tuple(cms) 

613 if self.mesh.mesh.comm.Get_rank() == 0: 

614 self.data_store.record_outputs("ROM", self.eval_idx, outputs) 

615 self.data_store.ROM_walltime += [(self.eval_idx, result.walltime)] 

616 self.data_store.rom_relative_residuals += [(self.eval_idx, result.eta)] 

617 self.data_store.ROM_force_coefficients += [(self.eval_idx, cd, cl) + cms] 

618 self.data_store.ROM_meets_threshold += [(self.eval_idx, result.accepted)] 

619 self.data_store.ROM_lspg_converged += [(self.eval_idx, result.converged)] 

620 PETSc.Sys.Print("ROM coefficients: c_d: {}, c_l: {}".format(cd, cl) + self._moment_log_string(cms)) 

621 return (cd, cl) + cms 

622 

623 def _log_rom_error(self, rom_forces): 

624 # ROM-vs-FOM coefficient errors against the FOM solve just run 

625 errors = tuple(float(r - f) for r, f in zip(rom_forces, self._last_fom_coefficients)) 

626 if self.mesh.mesh.comm.Get_rank() == 0: 

627 self.data_store.rom_coefficient_errors += [(self.eval_idx,) + errors] 

628 PETSc.Sys.Print("ROM - FOM coefficient errors: c_d: {:.3e}, c_l: {:.3e}".format(*errors[:2]) 

629 + "".join(", c_m_{}: {:.3e}".format(spec.name, e) 

630 for spec, e in zip(self.postprocessor.moment_specs, errors[2:]))) 

631 

632 def _evaluate_outputs(self, solution): 

633 """Every output of the postprocessor for a flow state: ({name: value}, 

634 (c_d, c_l, c_m_1, ...)). The names are DG_postprocessor.evaluate's (L, D, 

635 c_l, c_d, M_<name>, c_m_<name>); the tuple is the layout of the 

636 *_force_coefficients records, one moment per moment spec, in order.""" 

637 L, c_l = self.postprocessor.compute_cl(solution) 

638 D, c_d = self.postprocessor.compute_cd(solution) 

639 outputs = {"L": L, "D": D, "c_l": c_l, "c_d": c_d} 

640 cms = [] 

641 for spec in self.postprocessor.moment_specs: 

642 M, c_m = self.postprocessor.compute_moment(solution, spec) 

643 outputs["M_" + spec.name], outputs["c_m_" + spec.name] = M, c_m 

644 cms.append(c_m) 

645 return outputs, (c_d, c_l, *cms) 

646 

647 def _log_rom_state_error(self, solution_rom, solution_fom): 

648 # absolute and relative L2 errors of the ROM state against the FOM state just 

649 # run, over the domain and over the wall. Both are this rank's owned entries. 

650 if not self.mesh.mesh.comm.allreduce(bool(np.all(np.isfinite(solution_rom))), op=MPI.LAND): 

651 return 

652 diff = np.subtract(solution_fom, solution_rom) 

653 l2, l2_rel = self.sim_model.compute_L2_norm(diff, normalization_vec=solution_fom) 

654 l2_wall, l2_wall_rel = self.sim_model.compute_L2_norm( 

655 diff, normalization_vec=solution_fom, wall_tag=self.WALL_TAG) 

656 if self.mesh.mesh.comm.Get_rank() == 0: 

657 ds = self.data_store 

658 ds.absolute_l2_errors_per_iteration += [(self.eval_idx, l2)] 

659 ds.relative_l2_errors_per_iteration += [(self.eval_idx, l2_rel)] 

660 ds.absolute_l2_bdry_errors_per_iteration += [(self.eval_idx, l2_wall)] 

661 ds.relative_l2_bdry_errors_per_iteration += [(self.eval_idx, l2_wall_rel)] 

662 PETSc.Sys.Print("ROM L2 error relative to the FOM: domain {:.3e}, wall {:.3e}".format(l2_rel, l2_wall_rel)) 

663 

664 def _moment_log_string(self, coefficients): 

665 return "".join(", c_m_{}: {}".format(spec.name, c) 

666 for spec, c in zip(self.postprocessor.moment_specs, coefficients)) 

667 

668 def _run_fom_solve(self): 

669 # Run the FOM model and output its solution 

670 

671 # store pre-FOM time stamp for performance profiling 

672 fom_pre_time = perf_counter() 

673 

674 # use the most recent converged solution as warm start 

675 initial_condition = self.previous_solution 

676 self.sim_model.interpolate_solution_vector(initial_condition) 

677 

678 # solve the FOM 

679 # an incoming state that already meets the convergence criterion 

680 # needs no solver (SNES, KSP/PC, matrices): skip building it 

681 if self.sim_model.incoming_state_converged(): 

682 residual_norm = self.sim_model.residual_vec_physical.norm() 

683 else: 

684 self.sim_model.set_up_solver() 

685 residual_norm = self.sim_model.solve_system().norm() 

686 fom_converged = self.sim_model.solver_converged 

687 eta = self.sim_model.relative_residual_norm(residual_norm) 

688 solution_fom = self.sim_model.u_vec.x.petsc_vec.getArray().copy() 

689 if self.release_solver_memory: 

690 self.sim_model.release_solver() 

691 

692 self._log_memory("after FOM solve, before doing anything else") 

693 

694 # store post-FOM time stamp for performance profiling 

695 fom_post_time = perf_counter() 

696 PETSc.Sys.Print("FOM wall time: {}".format(fom_post_time - fom_pre_time)) 

697 

698 outputs_fom, (cd_fom, cl_fom, *cms_fom) = self._evaluate_outputs(solution_fom) 

699 cms_fom = tuple(cms_fom) 

700 self._last_fom_coefficients = (cd_fom, cl_fom) + cms_fom 

701 

702 if self.mesh.mesh.comm.Get_rank() == 0: 

703 self.data_store.record_outputs("FOM", self.eval_idx, outputs_fom) 

704 self.data_store.FOM_walltime += [(self.eval_idx, fom_post_time - fom_pre_time)] 

705 self.data_store.fom_relative_residuals += [(self.eval_idx, eta)] 

706 self.data_store.FOM_force_coefficients += [(self.eval_idx, cd_fom, cl_fom) + cms_fom] 

707 PETSc.Sys.Print("FOM coefficients: c_d: {}, c_l: {}".format(cd_fom, cl_fom) + self._moment_log_string(cms_fom)) 

708 PETSc.Sys.Print("FOM relative residual norm eta: {} (||r||: {})".format( 

709 eta, residual_norm)) 

710 

711 return solution_fom, fom_converged 

712 

713 def apply_inverse_jacobian(self, input_vals, outputs, d_outputs, d_residuals, mode): 

714 pre_apply_inverse_jacobian_time = perf_counter() 

715 # Here we solve the adjoint equation when calculating derivatives for optimization 

716 print("starting apply_inverse_jacobian in DG_windtunnel_model...") 

717 

718 mesh_node_motions = input_vals['mesh_node_motions'] if 'mesh_node_motions' in input_vals else None 

719 shape_parameters = input_vals['shape_parameters'] if 'shape_parameters' in input_vals else None 

720 alpha = input_vals['alpha'][0] 

721 u_vec = outputs['u_vec'] # numpy array 

722 

723 if mode == 'rev': 

724 # compute d_residual = (dr_du^-1)*d_outputs 

725 

726 # apply the input mesh node motions to the mesh of the sim model 

727 # and set the angle of attack before constructing the weak form 

728 # and the corresponding solution that we are linearizing around 

729 if mesh_node_motions is not None: 

730 PETSc.Sys.Print("apply_inverse_jacobian, stepping into mesh.apply_node_motions") 

731 self.mesh.apply_node_motions(mesh_node_motions, design=shape_parameters) 

732 self.sim_model.set_angle_of_attack(alpha) 

733 self.sim_model.compute_weakform() 

734 self.sim_model.set_u_vec(u_vec) 

735 

736 rom = self.reduced_order_model 

737 if (rom is not None and rom.adjoint is not None 

738 and rom.adjoint.matches(self._parameter_vector(shape_parameters, alpha), u_vec)): 

739 # The accepted solution is the ROM's: the adjoint of its 

740 # projected (LSPG) equations, lifted to the full state, which 

741 # compute_jacvec_product then uses like the full-order one 

742 d_residuals['u_vec'] = rom.adjoint.solve(self.sim_model, d_outputs['u_vec']) 

743 self._log_adjoint("rom", pre_apply_inverse_jacobian_time) 

744 if self.compute_all_adjoint_errors: 

745 # after the ROM adjoint's own timing; d_residuals is not touched 

746 self._log_adjoint_error(input_vals, u_vec, alpha, mesh_node_motions, 

747 d_outputs, d_residuals['u_vec']) 

748 return 

749 

750 self._fom_adjoint(input_vals, u_vec, alpha, mesh_node_motions, d_outputs, d_residuals, 

751 pre_apply_inverse_jacobian_time) 

752 else: 

753 raise ValueError("Reverse mode required") 

754 

755 def _fom_adjoint(self, input_vals, u_vec, alpha, mesh_node_motions, d_outputs, d_residuals, 

756 start_time, log=True): 

757 """The full-order adjoint psi = (dR/dU)^-T d_outputs['u_vec'] at the model's current 

758 state, written to d_residuals['u_vec']. log=False leaves it out of data_store.adjoint_solves 

759 (the reference solve of compute_all_adjoint_errors).""" 

760 if self.sim_model.has_condensed_jacobian: 

761 self._condensed_adjoint(d_outputs, d_residuals, 

762 self._adjoint_state_key(u_vec, alpha, mesh_node_motions)) 

763 if self.release_solver_memory: 

764 self.sim_model.release_solver() 

765 PETSc.garbage_cleanup() 

766 if log: 

767 self._log_adjoint("fom", start_time) 

768 return 

769 

770 self._drdu_mat = self.sim_model.assemble_dRdu_mat() 

771 if not hasattr(self, '_d_output_petsc'): 

772 self._d_output_petsc = self._drdu_mat.createVecLeft() 

773 self._d_residual_petsc = self._drdu_mat.createVecLeft() 

774 self._F_form = dolfinx.fem.form(self.sim_model.F) 

775 self._residual_vec_petsc = dolfinx.fem.petsc.create_vector( 

776 dolfinx.fem.extract_function_spaces(self._F_form)) 

777 

778 with self._residual_vec_petsc.localForm() as loc: 

779 loc.set(0.0) 

780 dolfinx.fem.petsc.assemble_vector(self._residual_vec_petsc, self._F_form) 

781 self._residual_vec_petsc.ghostUpdate(addv=PETSc.InsertMode.ADD, mode=PETSc.ScatterMode.REVERSE) 

782 PETSc.Sys.Print("max residual entry: {}".format(self._residual_vec_petsc.max()[1])) 

783 residual_norm = self._residual_vec_petsc.norm() 

784 PETSc.Sys.Print("forward eval residual norm: {}".format(residual_norm)) 

785 

786 # (no assembly here any more: assemble_dRdu_mat above already 

787 # returns the matrix assembled. It hands back the same cached 

788 # Mat object every call, so the KSP's operator reference below 

789 # stays valid.) 

790 if not hasattr(self, '_adj_solver'): 

791 self._adj_solver = PETSc.KSP().create(self.mesh.mesh.comm) 

792 self._adj_solver.setOperators(self._drdu_mat) 

793 self._adj_solver.setOptionsPrefix("adj_") 

794 

795 # define the adjoint solver settings 

796 self.sim_model.apply_krylov_solver_settings( 

797 self._adj_solver, max_it=self.linear_solver_max_it, 

798 gmres_restart=200, 

799 monitor_convergence=False, 

800 rtol=self.linear_solver_rtol, 

801 atol=self.linear_solver_atol) 

802 

803 set_petsc_vec_array(self._d_output_petsc, d_outputs['u_vec']) 

804 

805 # solve the transposed linear system and output the solution 

806 self._adj_solver.solveTranspose(self._d_output_petsc, self._d_residual_petsc) 

807 

808 rnorm = self._adj_solver.getResidualNorm() 

809 PETSc.Sys.Print("adj_ solve: {} iterations, residual norm {} [{}]".format( 

810 self._adj_solver.getIterationNumber(), rnorm, 

811 ksp_converged_reason_name(self._adj_solver.getConvergedReason()))) 

812 

813 if rnorm > self.linear_residual_accept_limit: 

814 raise ValueError( 

815 "apply_inverse_jacobian residual norm is too high: {:.6e} > " 

816 "linear_residual_accept_limit {:.1e}".format( 

817 rnorm, self.linear_residual_accept_limit)) 

818 

819 PETSc.Sys.Print("d_residual norm: {}".format(self._d_residual_petsc.norm())) 

820 

821 if mesh_node_motions is not None: 

822 PETSc.Sys.Print("mesh_node_motions min, max: {}, {}".format(mesh_node_motions.min(), mesh_node_motions.max())) 

823 PETSc.Sys.Print("shape parameters: {}".format(input_vals['shape_parameters'])) 

824 PETSc.Sys.Print("alpha: {}".format(np.degrees(alpha))) 

825 

826 d_residuals['u_vec'] = self._d_residual_petsc.getArray() 

827 

828 PETSc.garbage_cleanup() 

829 

830 if log: 

831 self._log_adjoint("fom", start_time) 

832 

833 def _log_adjoint_error(self, input_vals, u_vec, alpha, mesh_node_motions, d_outputs, psi_rom): 

834 """Record the error of the ROM adjoint psi_rom against the full-order adjoint at the 

835 same state, design and right-hand side: (eval_idx, output row, absolute 2-norm error, 

836 relative error, ||psi_fom||) in data_store.adjoint_errors. The row is the order of the 

837 adjoint solves within the evaluation, i.e. of CSDL's derivative rows. Collective.""" 

838 scratch = {} 

839 try: 

840 self._fom_adjoint(input_vals, u_vec, alpha, mesh_node_motions, d_outputs, scratch, 

841 perf_counter(), log=False) 

842 except ValueError as exc: 

843 # a diagnostic must not stop the optimization 

844 PETSc.Sys.Print("adjoint error: the full-order reference adjoint failed: {}".format(exc)) 

845 return 

846 # copy: the plain path returns a view of a PETSc vector reused by the next solve 

847 psi_fom = np.array(scratch['u_vec'], dtype=float) 

848 comm = self.mesh.mesh.comm 

849 diff = comm.allreduce(float(np.sum((np.asarray(psi_rom) - psi_fom)**2)), op=MPI.SUM) 

850 ref = comm.allreduce(float(psi_fom @ psi_fom), op=MPI.SUM) 

851 abs_err, fom_norm = np.sqrt(diff), np.sqrt(ref) 

852 rel_err = abs_err / fom_norm if fom_norm > 0.0 else np.nan 

853 

854 eval_idx = self.eval_idx - 1 

855 row = self._adjoint_row[1] if self._adjoint_row[0] == eval_idx else 0 

856 self._adjoint_row = (eval_idx, row + 1) 

857 if comm.Get_rank() == 0: 

858 self.data_store.adjoint_errors += [(eval_idx, row, float(abs_err), float(rel_err), float(fom_norm))] 

859 PETSc.Sys.Print("ROM adjoint error (output row {}): |psi_rom - psi_fom| = {:.3e}, relative {:.3e}".format( 

860 row, abs_err, rel_err)) 

861 

862 def _log_adjoint(self, kind, start_time): 

863 # which adjoint ('rom' or 'fom') the derivative of the last evaluation 

864 # used, and its wall time 

865 walltime = perf_counter() - start_time 

866 if self.mesh.mesh.comm.Get_rank() == 0: 

867 self.data_store.adjoint_solves += [(self.eval_idx - 1, kind, walltime)] 

868 PETSc.Sys.Print("apply_inverse_jacobian total time ({} adjoint): {}".format(kind, walltime)) 

869 

870 def _condensed_adjoint(self, d_outputs, d_residuals, state_key): 

871 """The adjoint solve for a model whose dR/dU is a condensed product 

872 (a reconstructed gradient): the model's own transposed linearized 

873 solve (CompressibleEulerModel.linearized_solver). The transposed 

874 matrix and the preconditioned KSP are built once per state and reused 

875 by the adjoint solves of all outputs at it.""" 

876 if self._adjoint_cache_matches(state_key): 

877 PETSc.Sys.Print("adj_: reusing the transposed Jacobian and preconditioner of this state") 

878 else: 

879 self._release_adjoint_cache() 

880 if self.release_solver_memory: 

881 # free queued PETSc objects before the large assembly 

882 gc.collect() 

883 PETSc.garbage_cleanup(self.mesh.mesh.comm) 

884 # the condensed dR/dU is freed once its transposes exist: only they are used from then on 

885 ksp = self.sim_model.linearized_solver( 

886 transpose=True, prefix="adj_", rtol=self.linear_solver_rtol, 

887 atol=self.linear_solver_atol, max_it=self.linear_solver_max_it, 

888 release_operands=True, matrix_free=self.adjoint_matrix_free, 

889 ksp_options=self.adjoint_ksp_options) 

890 self._adjoint_cache = {'key': state_key, 'ksp': ksp} 

891 ksp = self._adjoint_cache['ksp'] 

892 A = ksp.getOperators()[0] 

893 rhs, sol = A.createVecLeft(), A.createVecLeft() 

894 set_petsc_vec_array(rhs, d_outputs['u_vec']) 

895 guess_used = self.adjoint_first_order_guess and self._first_order_adjoint_guess(ksp, A, rhs, sol) 

896 ksp.setInitialGuessNonzero(bool(guess_used)) 

897 ksp.solve(rhs, sol) 

898 rnorm = ksp.getResidualNorm() 

899 PETSc.Sys.Print("adj_ solve: {} iterations, residual norm {} [{}]".format( 

900 ksp.getIterationNumber(), rnorm, ksp_converged_reason_name(ksp.getConvergedReason()))) 

901 if rnorm > self.linear_residual_accept_limit: 

902 raise ValueError( 

903 "apply_inverse_jacobian residual norm is too high: {:.6e} > " 

904 "linear_residual_accept_limit {:.1e}".format(rnorm, self.linear_residual_accept_limit)) 

905 d_residuals['u_vec'] = sol.getArray().copy() 

906 rhs.destroy() 

907 sol.destroy() 

908 

909 # ---- adjoint reuse: cached operators ----------------------------------- 

910 def _first_order_adjoint_guess(self, ksp, A, rhs, sol): 

911 """ 

912 Fill `sol` with the first-order adjoint solution of `rhs`: the solution 

913 of P x = rhs, P the transposed compact part A_UU of the Jacobian (the 

914 KSP's preconditioning matrix), to a relative tolerance 

915 first_order_adjoint_rtol. The KSP is switched to operator P for the 

916 solve, which keeps its preconditioner (P's factors), and back to 

917 (A, P) afterwards, with its tolerances. Returns False, leaving `sol` 

918 zero, if the first-order solve did not converge. The residual of the 

919 exact system at the guess, |rhs - A guess| / |rhs|, the iterations and 

920 the time are logged and kept in self._last_first_order_guess. 

921 Collective. 

922 """ 

923 P = ksp.getOperators()[1] 

924 if P.handle == A.handle: 

925 return False # direct solver: A is P, nothing to precondition with 

926 pre_time = perf_counter() 

927 rtol, atol, divtol, max_it = ksp.getTolerances() 

928 ksp.setOperators(P, P) 

929 ksp.setTolerances(rtol=self.first_order_adjoint_rtol, atol=0.0, max_it=self.linear_solver_max_it) 

930 ksp.setInitialGuessNonzero(False) 

931 try: 

932 ksp.solve(rhs, sol) 

933 iterations, reason = ksp.getIterationNumber(), ksp.getConvergedReason() 

934 finally: 

935 ksp.setOperators(A, P) 

936 ksp.setTolerances(rtol=rtol, atol=atol, divtol=divtol, max_it=max_it) 

937 if reason <= 0: 

938 PETSc.Sys.Print("adj_ first-order guess: no convergence after {} iterations [{}]: zero guess".format( 

939 iterations, ksp_converged_reason_name(reason))) 

940 sol.zeroEntries() 

941 self._last_first_order_guess = dict(iterations=iterations, ratio=np.nan, used=False) 

942 return False 

943 residual = rhs.duplicate() 

944 A.mult(sol, residual) 

945 residual.aypx(-1.0, rhs) 

946 ratio = residual.norm() / rhs.norm() 

947 residual.destroy() 

948 walltime = perf_counter() - pre_time 

949 PETSc.Sys.Print("adj_ first-order guess: {} iterations, {:.1f} s; the exact system's initial residual is " 

950 "{:.3e} of |rhs|".format(iterations, walltime, ratio)) 

951 self._last_first_order_guess = dict(iterations=iterations, ratio=float(ratio), used=True, walltime=walltime) 

952 return True 

953 

954 @staticmethod 

955 def _adjoint_state_key(u_vec, alpha, mesh_node_motions): 

956 """What the adjoint's matrix depends on: the state, alpha and the mesh motion.""" 

957 return (np.array(u_vec, copy=True), float(alpha), 

958 None if mesh_node_motions is None else np.array(mesh_node_motions, copy=True)) 

959 

960 def _state_keys_equal(self, key0, key): 

961 u0, alpha0, motions0 = key0 

962 u, alpha, motions = key 

963 same = (alpha == alpha0 and np.array_equal(u0, u) and (motions0 is None) == (motions is None) 

964 and (motions is None or np.array_equal(motions0, motions))) 

965 return bool(self.mesh.mesh.comm.allreduce(int(same), op=MPI.MIN)) 

966 

967 def _adjoint_cache_matches(self, state_key): 

968 cache = self._adjoint_cache 

969 return cache is not None and self._state_keys_equal(cache['key'], state_key) 

970 

971 def _release_adjoint_cache(self): 

972 """Free what the adjoint of the last state keeps: the transposed matrices 

973 and KSP, the cached jacvec operators and the derived fields' cached 

974 Jacobians.""" 

975 cache, self._adjoint_cache = self._adjoint_cache, None 

976 freed = cache is not None 

977 if freed: 

978 self.sim_model.destroy_adjoint_operator(cache['ksp']) 

979 del cache 

980 if self._dRdx_cache is not None: 

981 destroy_chained_operator(self._dRdx_cache['op']) 

982 self._dRdx_cache = None 

983 freed = True 

984 if self._dRdalpha_cache is not None: 

985 self._dRdalpha_cache = None 

986 if self.sim_model.derived_fields: 

987 self.sim_model.release_derived_caches() 

988 freed = True 

989 if freed: 

990 self.sim_model._return_freed_memory() 

991 

992 def set_adjoint_options(self, cell_centre_matrix=False, cache_jacobians=False, structured_gradient=False, 

993 matrix_free=False, cache_jacvec_operators=False): 

994 """ 

995 Switch the speed-ups of the adjoint of a model with a condensed Jacobian 

996 (all exact reformulations; each off by default). Frees every cache. 

997 cell_centre_matrix build the constant cell-centre matrix dc/dx vectorized, once 

998 cache_jacobians keep the Green-Gauss Jacobians dG/dU, dG/dalpha, dG/dx 

999 while the state, alpha and the mesh are unchanged 

1000 structured_gradient apply dG/dU as W (x) I plus a boundary part instead of an 

1001 assembled matrix 

1002 matrix_free the adjoint operator without forming A or A_UG B 

1003 cache_jacvec_operators build the dR/dx operator and dR/dalpha once per design 

1004 """ 

1005 self._release_adjoint_cache() 

1006 rec = getattr(self.sim_model, "reconstruction", None) 

1007 if rec is not None: 

1008 rec.fast_cell_centre_jacobian = bool(cell_centre_matrix) 

1009 rec.cache_jacobians = bool(cache_jacobians) 

1010 rec.structured_state_jacobian = bool(structured_gradient) 

1011 self.adjoint_matrix_free = bool(matrix_free) 

1012 self.cache_jacvec_operators = bool(cache_jacvec_operators) 

1013 

1014 def compute_jacvec_product(self, input_vals, outputs, d_inputs, d_outputs, d_residuals, mode): 

1015 # Here we compute the product of the adjoint solution with d_residual 

1016 pre_compute_jacvec_product_time = perf_counter() 

1017 print("starting compute_jacvec_product in DG_windtunnel_model...") 

1018 

1019 mesh_node_motions = input_vals['mesh_node_motions'] if 'mesh_node_motions' in input_vals else None 

1020 shape_parameters = input_vals['shape_parameters'] if 'shape_parameters' in input_vals else None 

1021 alpha = input_vals['alpha'][0] 

1022 u_vec = outputs['u_vec'] # numpy array 

1023 

1024 if mode == 'rev': 

1025 # compute d_input = (dr_dinput^T)*d_residual 

1026 

1027 # apply the input mesh node motions to the mesh of the sim model, 

1028 # apply the angle of attack and compute the weak form 

1029 if mesh_node_motions is not None: 

1030 PETSc.Sys.Print("compute_jacvec_product, stepping into mesh.apply_node_motions") 

1031 self.mesh.apply_node_motions(mesh_node_motions, design=shape_parameters) 

1032 self.sim_model.set_angle_of_attack(alpha) 

1033 self.sim_model.compute_weakform() 

1034 

1035 if 'alpha' in d_inputs: 

1036 # alpha is a single global scalar, so dR/d(alpha) is one column 

1037 # and this reverse product is a plain dot product between vectors 

1038 local_size = self.sim_model.u_vec.x.petsc_vec.local_size 

1039 if self.cache_jacvec_operators and self.sim_model.derived_fields: 

1040 key = self._adjoint_state_key(u_vec, alpha, mesh_node_motions) 

1041 if self._dRdalpha_cache is None or not self._state_keys_equal(self._dRdalpha_cache[0], key): 

1042 dr_dalpha = self.sim_model.assemble_dRdalpha_vec(u_vec) 

1043 self._dRdalpha_cache = (key, np.array(dr_dalpha.getArray()[:local_size], copy=True)) 

1044 dr_dalpha_local = self._dRdalpha_cache[1] 

1045 else: 

1046 dr_dalpha = self.sim_model.assemble_dRdalpha_vec(u_vec) 

1047 

1048 # we first have to assert that the MPI objects are compatible 

1049 dr_dalpha_local = dr_dalpha.getArray()[:local_size] 

1050 assert dr_dalpha_local.shape == d_residuals['u_vec'].shape, ( 

1051 "dR/dalpha local slice {} does not match d_residuals['u_vec'] {}".format( 

1052 dr_dalpha_local.shape, d_residuals['u_vec'].shape)) 

1053 

1054 # we do the multiplication of the local objects through Numpy, 

1055 # after which we sum the contributions on every MPI rank 

1056 local_contribution = float(np.dot(dr_dalpha_local, d_residuals['u_vec'])) 

1057 d_inputs['alpha'] = np.array( 

1058 [self.mesh.mesh.comm.allreduce(local_contribution, op=MPI.SUM)], 

1059 dtype=np.float64) 

1060 

1061 PETSc.Sys.Print("d_inputs['alpha']: {}".format(d_inputs['alpha'])) 

1062 

1063 if 'mesh_node_motions' in d_inputs: 

1064 cached_operator = self.cache_jacvec_operators and bool(self.sim_model.derived_fields) 

1065 if cached_operator: 

1066 key = self._adjoint_state_key(u_vec, alpha, mesh_node_motions) 

1067 if self._dRdx_cache is None or not self._state_keys_equal(self._dRdx_cache['key'], key): 

1068 if self._dRdx_cache is not None: 

1069 destroy_chained_operator(self._dRdx_cache['op']) 

1070 self._dRdx_cache = {'key': key, 'op': self.sim_model.compute_dRdxmesh_mat( 

1071 u_vec, self.mesh.coordinate_space)} 

1072 dr_dxmesh = self._dRdx_cache['op'] 

1073 else: 

1074 dr_dxmesh = self.sim_model.compute_dRdxmesh_mat(u_vec, self.mesh.coordinate_space) 

1075 

1076 if not hasattr(self, '_d_residual_jvp_petsc'): 

1077 self._d_residual_jvp_petsc = dr_dxmesh.createVecLeft() 

1078 self._d_input_jvp_petsc = dr_dxmesh.createVecRight() 

1079 

1080 set_petsc_vec_array(self._d_residual_jvp_petsc, d_residuals['u_vec']) 

1081 

1082 # compute d_input 

1083 self._d_input_jvp_petsc.set(0.0) 

1084 dr_dxmesh.multTranspose(self._d_residual_jvp_petsc, self._d_input_jvp_petsc) 

1085 d_input = self._d_input_jvp_petsc 

1086 if self.sim_model.derived_fields and not cached_operator: 

1087 # a fresh chained operator per call (not cached): free it now 

1088 destroy_chained_operator(dr_dxmesh) 

1089 

1090 PETSc.Sys.Print("d_input norm: {}".format(d_input.norm())) 

1091 

1092 d_input_cg1 = d_input.getArray().reshape(-1, self.mesh.mesh.geometry.dim) # (N_CG1_dofs, gdim) 

1093 d_input_geom = self.mesh.dof_to_node_map.T @ d_input_cg1 # (N_geom_nodes, gdim) 

1094 d_inputs['mesh_node_motions'] = d_input_geom 

1095 

1096 PETSc.garbage_cleanup() 

1097 else: 

1098 raise ValueError("Reverse mode required") 

1099 

1100 PETSc.Sys.Print("DG_windtunnel_model compute_jacvec_product done") 

1101 

1102 post_compute_jacvec_product_time = perf_counter() 

1103 total_compute_jacvec_product_time = post_compute_jacvec_product_time - pre_compute_jacvec_product_time 

1104 

1105 PETSc.Sys.Print("compute_jacvec_product total time: {}".format(total_compute_jacvec_product_time))