Line data Source code
1 : use std::convert::Infallible;
2 : use std::sync::{Arc, Mutex};
3 :
4 : use crossbeam_skiplist::SkipMap;
5 : use crossbeam_skiplist::equivalent::{Comparable, Equivalent};
6 : use moka::sync::Cache;
7 : use tracing::{debug, info};
8 :
9 : use crate::cache::common::{
10 : ControlPlaneResult, CplaneExpiry, count_cache_insert, count_cache_outcome, eviction_listener,
11 : };
12 : use crate::config::ProjectInfoCacheOptions;
13 : use crate::control_plane::messages::{ControlPlaneErrorMessage, Reason};
14 : use crate::control_plane::{EndpointAccessControl, RoleAccessControl};
15 : use crate::ext::LockExt;
16 : use crate::intern::{AccountIdInt, EndpointIdInt, ProjectIdInt, RoleNameInt};
17 : use crate::metrics::{CacheKind, Metrics};
18 : use crate::types::{EndpointId, RoleName};
19 :
20 : /// Cache for project info.
21 : /// This is used to cache auth data for endpoints.
22 : /// Invalidation is done by console notifications or by TTL (if console notifications are disabled).
23 : ///
24 : /// We also store endpoint-to-project mapping in the cache, to be able to access per-endpoint data.
25 : /// One may ask, why the data is stored per project, when on the user request there is only data about the endpoint available?
26 : /// On the cplane side updates are done per project (or per branch), so it's easier to invalidate the whole project cache.
27 : pub struct ProjectInfoCache {
28 : role_controls:
29 : Cache<(EndpointIdInt, RoleNameInt), ControlPlaneResult<Entry<RoleAccessControl>>>,
30 : ep_controls: Cache<EndpointIdInt, ControlPlaneResult<Entry<EndpointAccessControl>>>,
31 :
32 : project2ep: Arc<RefCountMultiSet<ProjectIdInt, EndpointIdInt>>,
33 : account2ep: Arc<RefCountMultiSet<AccountIdInt, EndpointIdInt>>,
34 :
35 : config: ProjectInfoCacheOptions,
36 : }
37 :
38 : type RefCount = Mutex<usize>;
39 :
40 : // This is rather hacky.
41 : // We use an ordered map of (K, V) -> RefCount.
42 : // We use range queries over `(K, _)..(K+1, _)` to do the invalidation.
43 : // We use the RefCount to know when to remove entries.
44 : type RefCountMultiSet<K, V> = SkipMap<KeyValue<K, V>, RefCount>;
45 :
46 : #[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Debug)]
47 : struct KeyValue<K, V>(K, V);
48 : struct Key<'a, K>(&'a K, bool);
49 :
50 : impl<'a, K> Key<'a, K> {
51 0 : fn prefix(key: &'a K) -> std::ops::Range<Self> {
52 0 : Self(key, false)..Self(key, true)
53 0 : }
54 : }
55 :
56 : impl<'a, K: Ord, V> Equivalent<Key<'a, K>> for KeyValue<K, V> {
57 0 : fn equivalent(&self, key: &Key<'a, K>) -> bool {
58 0 : self.0 == *key.0 && !key.1
59 0 : }
60 : }
61 : impl<'a, K: Ord, V> Comparable<Key<'a, K>> for KeyValue<K, V> {
62 0 : fn compare(&self, key: &Key<'a, K>) -> std::cmp::Ordering {
63 0 : self.0.cmp(key.0).then(false.cmp(&key.1))
64 0 : }
65 : }
66 :
67 : #[derive(Clone)]
68 : struct Entry<T> {
69 : project_id: Option<ProjectIdInt>,
70 : account_id: Option<AccountIdInt>,
71 : value: T,
72 : }
73 :
74 : impl<T> Entry<T> {
75 7 : fn dec_ref_counts(
76 7 : self,
77 7 : project2ep: &RefCountMultiSet<ProjectIdInt, EndpointIdInt>,
78 7 : account2ep: &RefCountMultiSet<AccountIdInt, EndpointIdInt>,
79 7 : endpoint_id: EndpointIdInt,
80 7 : ) {
81 7 : if let Some(project_id) = self.project_id {
82 7 : dec_ref_count(project2ep, project_id, endpoint_id);
83 7 : }
84 7 : if let Some(account_id) = self.account_id {
85 0 : dec_ref_count(account2ep, account_id, endpoint_id);
86 7 : }
87 7 : }
88 : }
89 :
90 7 : fn dec_ref_count<Id: Ord + Send + 'static>(
91 7 : id2ep: &RefCountMultiSet<Id, EndpointIdInt>,
92 7 : id: Id,
93 7 : endpoint_id: EndpointIdInt,
94 7 : ) {
95 7 : if let Some(entry) = id2ep.get(&KeyValue(id, endpoint_id)) {
96 7 : let mut count = entry.value().lock_propagate_poison();
97 7 : *count -= 1;
98 7 : if *count == 0 {
99 1 : // remove the entry while holding the lock
100 1 : entry.remove();
101 6 : }
102 0 : }
103 7 : }
104 :
105 : impl ProjectInfoCache {
106 0 : pub fn invalidate_endpoint_access(&self, endpoint_id: EndpointIdInt) {
107 0 : info!("invalidating endpoint access for `{endpoint_id}`");
108 0 : self.ep_controls.invalidate(&endpoint_id);
109 0 : }
110 :
111 0 : pub fn invalidate_endpoint_access_for_project(&self, project_id: ProjectIdInt) {
112 0 : info!("invalidating endpoint access for project `{project_id}`");
113 :
114 0 : for entry in self.project2ep.range(Key::prefix(&project_id)) {
115 0 : self.ep_controls.invalidate(&entry.key().1);
116 0 : }
117 0 : }
118 :
119 0 : pub fn invalidate_endpoint_access_for_org(&self, account_id: AccountIdInt) {
120 0 : info!("invalidating endpoint access for org `{account_id}`");
121 :
122 0 : for entry in self.account2ep.range(Key::prefix(&account_id)) {
123 0 : self.ep_controls.invalidate(&entry.key().1);
124 0 : }
125 0 : }
126 :
127 0 : pub fn invalidate_role_secret_for_project(
128 0 : &self,
129 0 : project_id: ProjectIdInt,
130 0 : role_name: RoleNameInt,
131 0 : ) {
132 0 : info!(
133 0 : "invalidating role secret for project_id `{}` and role_name `{}`",
134 : project_id, role_name,
135 : );
136 :
137 0 : for entry in self.project2ep.range(Key::prefix(&project_id)) {
138 0 : self.role_controls.invalidate(&(entry.key().1, role_name));
139 0 : }
140 0 : }
141 : }
142 :
143 : impl ProjectInfoCache {
144 2 : pub(crate) fn new(config: ProjectInfoCacheOptions) -> Self {
145 2 : Metrics::get().cache.capacity.set(
146 2 : CacheKind::ProjectInfoRoles,
147 2 : (config.size * config.max_roles) as i64,
148 : );
149 2 : Metrics::get()
150 2 : .cache
151 2 : .capacity
152 2 : .set(CacheKind::ProjectInfoEndpoints, config.size as i64);
153 :
154 2 : let project2ep = Arc::new(RefCountMultiSet::<ProjectIdInt, EndpointIdInt>::new());
155 2 : let account2ep = Arc::new(RefCountMultiSet::<AccountIdInt, EndpointIdInt>::new());
156 2 : let project2ep1 = Arc::clone(&project2ep);
157 2 : let project2ep2 = Arc::clone(&project2ep);
158 2 : let account2ep1 = Arc::clone(&account2ep);
159 2 : let account2ep2 = Arc::clone(&account2ep);
160 :
161 : // we cache errors for 30 seconds, unless retry_at is set.
162 2 : let expiry = CplaneExpiry::default();
163 : Self {
164 2 : role_controls: Cache::builder()
165 2 : .name("role_access_controls")
166 2 : .eviction_listener(
167 5 : move |k, v: ControlPlaneResult<Entry<RoleAccessControl>>, cause| {
168 5 : eviction_listener(CacheKind::ProjectInfoRoles, cause);
169 :
170 5 : let (endpoint_id, _): (EndpointIdInt, RoleNameInt) = *k;
171 5 : if let Ok(v) = v {
172 4 : v.dec_ref_counts(&project2ep1, &account2ep1, endpoint_id);
173 4 : }
174 5 : },
175 : )
176 2 : .max_capacity(config.size * config.max_roles)
177 2 : .time_to_live(config.ttl)
178 2 : .expire_after(expiry)
179 2 : .build(),
180 2 : ep_controls: Cache::builder()
181 2 : .name("endpoint_access_controls")
182 2 : .eviction_listener(
183 4 : move |k, v: ControlPlaneResult<Entry<EndpointAccessControl>>, cause| {
184 4 : eviction_listener(CacheKind::ProjectInfoEndpoints, cause);
185 :
186 4 : let endpoint_id: EndpointIdInt = *k;
187 4 : if let Ok(v) = v {
188 3 : v.dec_ref_counts(&project2ep2, &account2ep2, endpoint_id);
189 3 : }
190 4 : },
191 : )
192 2 : .max_capacity(config.size)
193 2 : .time_to_live(config.ttl)
194 2 : .expire_after(expiry)
195 2 : .build(),
196 2 : project2ep,
197 2 : account2ep,
198 2 : config,
199 : }
200 2 : }
201 :
202 8 : pub(crate) fn get_role_secret(
203 8 : &self,
204 8 : endpoint_id: &EndpointId,
205 8 : role_name: &RoleName,
206 8 : ) -> Option<ControlPlaneResult<RoleAccessControl>> {
207 8 : let endpoint_id = EndpointIdInt::get(endpoint_id)?;
208 8 : let role_name = RoleNameInt::get(role_name)?;
209 :
210 7 : count_cache_outcome(
211 7 : CacheKind::ProjectInfoRoles,
212 7 : self.role_controls
213 7 : .get(&(endpoint_id, role_name))
214 7 : .map(|e| e.map(|e| e.value)),
215 : )
216 8 : }
217 :
218 4 : pub(crate) fn get_endpoint_access(
219 4 : &self,
220 4 : endpoint_id: &EndpointId,
221 4 : ) -> Option<ControlPlaneResult<EndpointAccessControl>> {
222 4 : let endpoint_id = EndpointIdInt::get(endpoint_id)?;
223 :
224 4 : count_cache_outcome(
225 4 : CacheKind::ProjectInfoEndpoints,
226 4 : self.ep_controls
227 4 : .get(&endpoint_id)
228 4 : .map(|e| e.map(|e| e.value)),
229 : )
230 4 : }
231 :
232 4 : pub(crate) fn insert_endpoint_access(
233 4 : &self,
234 4 : account_id: Option<AccountIdInt>,
235 4 : project_id: Option<ProjectIdInt>,
236 4 : endpoint_id: EndpointIdInt,
237 4 : role_name: RoleNameInt,
238 4 : controls: EndpointAccessControl,
239 4 : role_controls: RoleAccessControl,
240 4 : ) {
241 : // 2 corresponds to how many cache inserts we do.
242 4 : if let Some(account_id) = account_id {
243 0 : self.inc_account2ep_ref(account_id, endpoint_id, 2);
244 4 : }
245 4 : if let Some(project_id) = project_id {
246 4 : self.inc_project2ep_ref(project_id, endpoint_id, 2);
247 4 : }
248 :
249 4 : debug!(
250 0 : key = &*endpoint_id,
251 0 : "created a cache entry for endpoint access"
252 : );
253 :
254 4 : count_cache_insert(CacheKind::ProjectInfoEndpoints);
255 4 : count_cache_insert(CacheKind::ProjectInfoRoles);
256 :
257 4 : self.ep_controls.insert(
258 4 : endpoint_id,
259 4 : Ok(Entry {
260 4 : account_id,
261 4 : project_id,
262 4 : value: controls,
263 4 : }),
264 : );
265 4 : self.role_controls.insert(
266 4 : (endpoint_id, role_name),
267 4 : Ok(Entry {
268 4 : account_id,
269 4 : project_id,
270 4 : value: role_controls,
271 4 : }),
272 : );
273 4 : }
274 :
275 3 : pub(crate) fn insert_endpoint_access_err(
276 3 : &self,
277 3 : endpoint_id: EndpointIdInt,
278 3 : role_name: RoleNameInt,
279 3 : msg: Box<ControlPlaneErrorMessage>,
280 3 : ) {
281 3 : debug!(
282 0 : key = &*endpoint_id,
283 0 : "created a cache entry for an endpoint access error"
284 : );
285 :
286 : // RoleProtected is the only role-specific error that control plane can give us.
287 : // If a given role name does not exist, it still returns a successful response,
288 : // just with an empty secret.
289 3 : if msg.get_reason() != Reason::RoleProtected {
290 : // We can cache all the other errors in ep_controls because they don't
291 : // depend on what role name we pass to control plane.
292 2 : self.ep_controls
293 2 : .entry(endpoint_id)
294 2 : .and_compute_with(|entry| match entry {
295 : // leave the entry alone if it's already Ok
296 1 : Some(entry) if entry.value().is_ok() => moka::ops::compute::Op::Nop,
297 : // replace the entry
298 : _ => {
299 1 : count_cache_insert(CacheKind::ProjectInfoEndpoints);
300 1 : moka::ops::compute::Op::Put(Err(msg.clone()))
301 : }
302 2 : });
303 1 : }
304 :
305 3 : count_cache_insert(CacheKind::ProjectInfoRoles);
306 3 : self.role_controls
307 3 : .insert((endpoint_id, role_name), Err(msg));
308 3 : }
309 :
310 4 : fn inc_project2ep_ref(&self, project_id: ProjectIdInt, endpoint_id: EndpointIdInt, x: usize) {
311 4 : let entry = self
312 4 : .project2ep
313 4 : .get_or_insert(KeyValue(project_id, endpoint_id), Mutex::new(0));
314 4 : *entry.value().lock_propagate_poison() += x;
315 4 : }
316 :
317 0 : fn inc_account2ep_ref(&self, account_id: AccountIdInt, endpoint_id: EndpointIdInt, x: usize) {
318 0 : let entry = self
319 0 : .account2ep
320 0 : .get_or_insert(KeyValue(account_id, endpoint_id), Mutex::new(0));
321 0 : *entry.value().lock_propagate_poison() += x;
322 0 : }
323 :
324 0 : pub fn maybe_invalidate_role_secret(&self, _endpoint_id: &EndpointId, _role_name: &RoleName) {
325 : // TODO: Expire the value early if the key is idle.
326 : // Currently not an issue as we would just use the TTL to decide, which is what already happens.
327 0 : }
328 :
329 0 : pub async fn gc_worker(&self) -> anyhow::Result<Infallible> {
330 0 : let mut interval = tokio::time::interval(self.config.gc_interval);
331 : loop {
332 0 : interval.tick().await;
333 0 : self.ep_controls.run_pending_tasks();
334 0 : self.role_controls.run_pending_tasks();
335 : }
336 : }
337 : }
338 :
339 : #[cfg(test)]
340 : mod tests {
341 : use std::sync::Arc;
342 : use std::time::Duration;
343 :
344 : use super::*;
345 : use crate::control_plane::messages::{Details, EndpointRateLimitConfig, ErrorInfo, Status};
346 : use crate::control_plane::{AccessBlockerFlags, AuthSecret};
347 : use crate::scram::ServerSecret;
348 :
349 : #[tokio::test]
350 1 : async fn test_project_info_cache_settings() {
351 1 : let cache = ProjectInfoCache::new(ProjectInfoCacheOptions {
352 1 : size: 1,
353 1 : max_roles: 2,
354 1 : ttl: Duration::from_secs(1),
355 1 : gc_interval: Duration::from_secs(600),
356 1 : });
357 1 : let project_id: Option<ProjectIdInt> = Some(ProjectIdInt::from(&"project".into()));
358 1 : let endpoint_id: EndpointId = "endpoint".into();
359 1 : let account_id = None;
360 :
361 1 : let user1: RoleName = "user1".into();
362 1 : let user2: RoleName = "user2".into();
363 1 : let secret1 = Some(AuthSecret::Scram(ServerSecret::mock([1; 32])));
364 1 : let secret2 = None;
365 1 : let allowed_ips = Arc::new(vec![
366 1 : "127.0.0.1".parse().unwrap(),
367 1 : "127.0.0.2".parse().unwrap(),
368 : ]);
369 :
370 1 : cache.insert_endpoint_access(
371 1 : account_id,
372 1 : project_id,
373 1 : (&endpoint_id).into(),
374 1 : (&user1).into(),
375 1 : EndpointAccessControl {
376 1 : allowed_ips: allowed_ips.clone(),
377 1 : allowed_vpce: Arc::new(vec![]),
378 1 : flags: AccessBlockerFlags::default(),
379 1 : rate_limits: EndpointRateLimitConfig::default(),
380 1 : },
381 1 : RoleAccessControl {
382 1 : secret: secret1.clone(),
383 1 : },
384 : );
385 :
386 1 : cache.ep_controls.run_pending_tasks();
387 1 : cache.role_controls.run_pending_tasks();
388 :
389 : // check the project mappings are there
390 1 : assert_eq!(cache.project2ep.len(), 1);
391 :
392 : // check the ref counts
393 1 : let entry = cache.project2ep.front().unwrap();
394 1 : assert_eq!(*entry.value().lock_propagate_poison(), 2);
395 :
396 1 : cache.insert_endpoint_access(
397 1 : account_id,
398 1 : project_id,
399 1 : (&endpoint_id).into(),
400 1 : (&user2).into(),
401 1 : EndpointAccessControl {
402 1 : allowed_ips: allowed_ips.clone(),
403 1 : allowed_vpce: Arc::new(vec![]),
404 1 : flags: AccessBlockerFlags::default(),
405 1 : rate_limits: EndpointRateLimitConfig::default(),
406 1 : },
407 1 : RoleAccessControl {
408 1 : secret: secret2.clone(),
409 1 : },
410 : );
411 :
412 1 : cache.ep_controls.run_pending_tasks();
413 1 : cache.role_controls.run_pending_tasks();
414 :
415 : // check the project mappings are still there
416 1 : assert_eq!(cache.project2ep.len(), 1);
417 :
418 : // check the ref counts
419 1 : let entry = cache.project2ep.front().unwrap();
420 1 : assert_eq!(*entry.value().lock_propagate_poison(), 3);
421 :
422 : // check both entries exist
423 1 : let cached = cache.get_role_secret(&endpoint_id, &user1).unwrap();
424 1 : assert_eq!(cached.unwrap().secret, secret1);
425 :
426 1 : let cached = cache.get_role_secret(&endpoint_id, &user2).unwrap();
427 1 : assert_eq!(cached.unwrap().secret, secret2);
428 :
429 : // Shouldn't add more than 2 roles.
430 1 : let user3: RoleName = "user3".into();
431 1 : let secret3 = Some(AuthSecret::Scram(ServerSecret::mock([3; 32])));
432 :
433 1 : cache.role_controls.run_pending_tasks();
434 1 : cache.insert_endpoint_access(
435 1 : account_id,
436 1 : project_id,
437 1 : (&endpoint_id).into(),
438 1 : (&user3).into(),
439 1 : EndpointAccessControl {
440 1 : allowed_ips: allowed_ips.clone(),
441 1 : allowed_vpce: Arc::new(vec![]),
442 1 : flags: AccessBlockerFlags::default(),
443 1 : rate_limits: EndpointRateLimitConfig::default(),
444 1 : },
445 1 : RoleAccessControl {
446 1 : secret: secret3.clone(),
447 1 : },
448 : );
449 :
450 1 : cache.ep_controls.run_pending_tasks();
451 1 : cache.role_controls.run_pending_tasks();
452 :
453 1 : assert_eq!(cache.role_controls.entry_count(), 2);
454 :
455 : // check the project mappings are still there
456 1 : assert_eq!(cache.project2ep.len(), 1);
457 :
458 : // check the ref counts are unchanged.
459 1 : let entry = cache.project2ep.front().unwrap();
460 1 : assert_eq!(*entry.value().lock_propagate_poison(), 3);
461 :
462 1 : tokio::time::sleep(Duration::from_secs(2)).await;
463 :
464 1 : cache.ep_controls.run_pending_tasks();
465 1 : cache.role_controls.run_pending_tasks();
466 1 : assert_eq!(cache.role_controls.entry_count(), 0);
467 :
468 : // check the project/account mappings are no longer there
469 1 : assert!(cache.project2ep.is_empty());
470 1 : }
471 :
472 : #[tokio::test]
473 1 : async fn test_caching_project_info_errors() {
474 1 : let cache = ProjectInfoCache::new(ProjectInfoCacheOptions {
475 1 : size: 10,
476 1 : max_roles: 10,
477 1 : ttl: Duration::from_secs(1),
478 1 : gc_interval: Duration::from_secs(600),
479 1 : });
480 1 : let project_id = Some(ProjectIdInt::from(&"project".into()));
481 1 : let endpoint_id: EndpointId = "endpoint".into();
482 1 : let account_id = None;
483 :
484 1 : let user1: RoleName = "user1".into();
485 1 : let user2: RoleName = "user2".into();
486 1 : let secret = Some(AuthSecret::Scram(ServerSecret::mock([1; 32])));
487 :
488 1 : let role_msg = Box::new(ControlPlaneErrorMessage {
489 1 : error: "role is protected and cannot be used for password-based authentication"
490 1 : .to_owned()
491 1 : .into_boxed_str(),
492 1 : http_status_code: http::StatusCode::NOT_FOUND,
493 1 : status: Some(Status {
494 1 : code: "PERMISSION_DENIED".to_owned().into_boxed_str(),
495 1 : message: "role is protected and cannot be used for password-based authentication"
496 1 : .to_owned()
497 1 : .into_boxed_str(),
498 1 : details: Details {
499 1 : error_info: Some(ErrorInfo {
500 1 : reason: Reason::RoleProtected,
501 1 : }),
502 1 : retry_info: None,
503 1 : user_facing_message: None,
504 1 : },
505 1 : }),
506 1 : });
507 :
508 1 : let generic_msg = Box::new(ControlPlaneErrorMessage {
509 1 : error: "oh noes".to_owned().into_boxed_str(),
510 1 : http_status_code: http::StatusCode::NOT_FOUND,
511 1 : status: None,
512 1 : });
513 :
514 1 : let get_role_secret =
515 5 : |endpoint_id, role_name| cache.get_role_secret(endpoint_id, role_name).unwrap();
516 3 : let get_endpoint_access = |endpoint_id| cache.get_endpoint_access(endpoint_id).unwrap();
517 :
518 : // stores role-specific errors only for get_role_secret
519 1 : cache.insert_endpoint_access_err((&endpoint_id).into(), (&user1).into(), role_msg.clone());
520 1 : assert_eq!(
521 1 : get_role_secret(&endpoint_id, &user1).unwrap_err().error,
522 : role_msg.error
523 : );
524 1 : assert!(cache.get_endpoint_access(&endpoint_id).is_none());
525 :
526 : // stores non-role specific errors for both get_role_secret and get_endpoint_access
527 1 : cache.insert_endpoint_access_err(
528 1 : (&endpoint_id).into(),
529 1 : (&user1).into(),
530 1 : generic_msg.clone(),
531 : );
532 1 : assert_eq!(
533 1 : get_role_secret(&endpoint_id, &user1).unwrap_err().error,
534 : generic_msg.error
535 : );
536 1 : assert_eq!(
537 1 : get_endpoint_access(&endpoint_id).unwrap_err().error,
538 : generic_msg.error
539 : );
540 :
541 : // error isn't returned for other roles in the same endpoint
542 1 : assert!(cache.get_role_secret(&endpoint_id, &user2).is_none());
543 :
544 : // success for a role does not overwrite errors for other roles
545 1 : cache.insert_endpoint_access(
546 1 : account_id,
547 1 : project_id,
548 1 : (&endpoint_id).into(),
549 1 : (&user2).into(),
550 1 : EndpointAccessControl {
551 1 : allowed_ips: Arc::new(vec![]),
552 1 : allowed_vpce: Arc::new(vec![]),
553 1 : flags: AccessBlockerFlags::default(),
554 1 : rate_limits: EndpointRateLimitConfig::default(),
555 1 : },
556 1 : RoleAccessControl {
557 1 : secret: secret.clone(),
558 1 : },
559 : );
560 1 : assert!(get_role_secret(&endpoint_id, &user1).is_err());
561 1 : assert!(get_role_secret(&endpoint_id, &user2).is_ok());
562 : // ...but does clear the access control error
563 1 : assert!(get_endpoint_access(&endpoint_id).is_ok());
564 :
565 : // storing an error does not overwrite successful access control response
566 1 : cache.insert_endpoint_access_err(
567 1 : (&endpoint_id).into(),
568 1 : (&user2).into(),
569 1 : generic_msg.clone(),
570 : );
571 1 : assert!(get_role_secret(&endpoint_id, &user2).is_err());
572 1 : assert!(get_endpoint_access(&endpoint_id).is_ok());
573 1 : }
574 : }
|
